patchworks 2.6.11__tar.gz → 2.7.0__tar.gz

{patchworks-2.6.11 → patchworks-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: patchworks
-Version: 2.6.11
+Version: 2.7.0
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues
@@ -57,6 +57,7 @@ Requires-Dist: cellpose<4,>=3.0; extra == 'cellpose3'
 Provides-Extra: cellpose4
 Requires-Dist: cellpose>=4; extra == 'cellpose4'
 Provides-Extra: dev
+Requires-Dist: jsonschema>=4.18; extra == 'dev'
 Requires-Dist: psutil; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
 Requires-Dist: pytest-cov; extra == 'dev'

{patchworks-2.6.11 → patchworks-2.7.0}/docs/api/plugins/ome_zarr.md

@@ -25,6 +25,10 @@ resolution, matching anisotropic microscopy stacks.
 
 ::: patchworks.plugins.ome_zarr.register_labels
 
+## reshard_level
+
+::: patchworks.plugins.ome_zarr.reshard_level
+
 ## read_pixel_size
 
 ::: patchworks.plugins.ome_zarr.read_pixel_size
@@ -37,6 +41,14 @@ matches the store, and reads both.
 
 ::: patchworks.plugins.ome_zarr.ngff_version
 
+Every writer takes an `ngff_version=` keyword: `"auto"` (default) follows the
+installed zarr — 0.5 on v3, 0.4 on v2 — and `"0.4"` pins the older, zarr-v2
+layout. Writing into an existing store always follows *that store's* format,
+so a label pyramid added later can never disagree with the image it sits in.
+NGFF 0.6 is released but not written: RFC-5 replaces `axes` with
+`coordinateSystems` and requires `input`/`output` on every coordinate
+transformation, and no reader supports it yet.
+
 ::: patchworks.plugins.ome_zarr.read_ngff_attr
 
 ::: patchworks.plugins.ome_zarr.write_ngff_attrs

{patchworks-2.6.11 → patchworks-2.7.0}/docs/guide/snakemake.md

@@ -50,7 +50,9 @@ work_dir: "/scratch/results"   # everything is written here
 # conversion (input → pyramidal OME-ZARR)
 reuse_pyramid: true            # .ims: copy its own pyramid (fast)
 convert_chunks: null           # null → bounded auto chunks; or [c,z,y,x]
-shard: false                   # true → pack chunks into shards (fewer files)
+shard: false                   # true → pack chunks into shards (fewer files);
+                               # covers the image and the label pyramids
+ngff_version: "auto"           # OME-ZARR version: "auto" (0.5), or "0.4"
 
 # tiling
 channel: 0                     # channel to segment, 0-based (null = keep all)
@@ -83,6 +85,8 @@ cellpose:
 pyramid_levels: 5
 pyramid_downscale: 2
 sequential_labels: true        # renumber labels to a contiguous 1..N
+shard_labels: false            # true → also reshard label level 0 after the
+                               # merge (one extra pass; see tip below)
 ```
 
 !!! tip "Growing labels after segmentation"
@@ -98,6 +102,77 @@ sequential_labels: true        # renumber labels to a contiguous 1..N
     labels afterwards](custom_segmentation.md#growing-labels-afterwards-dilation)
     for how it works and the equivalent direct-API call.
 
+!!! tip "What `shard: true` covers, and `shard_labels` for label level 0"
+    Without sharding, one chunk is one file, and a fine-chunked level 0 can
+    run to ~950k of them — painful on a shared filesystem at write time and
+    on every read after. `shard: true` packs chunks into far fewer files
+    without changing the chunking or the memory profile.
+
+    `shard: true` applies to **the converted image (all levels)** and to
+    **the label pyramid levels (1..N)**. It cannot apply to **label level
+    0** *while that level is being written*: a shard has to be written whole
+    by a single writer, while level 0 is filled one chunk at a time by
+    concurrent `segment` jobs (or by the merge's own worker pool). Two
+    writers doing a read-modify-write on the same shard file silently lose
+    each other's chunks.
+
+    Once the merge has finished, though, nothing is writing it anymore, so a
+    single-threaded pass *can* rewrite it sharded. That is `shard_labels`:
+
+    ```yaml
+    shard: true
+    shard_labels: true       # or an explicit shape, e.g. [16, 512, 512]
+    ```
+
+    `true` reuses whatever spec `shard` carries; a list overrides it.
+    `shard: false` does not veto it — an unsharded image with sharded labels
+    is a valid combination. It is opt-in because it costs one extra full
+    read+write of level 0, and because it is the level with the most chunks
+    it is also the one worth paying for. The array's attributes (including
+    the merge's own completion marker) are carried across, so a later rerun
+    still sees the merge as done.
+
+    Check whether the file count is actually a problem for your data first —
+    a `(126, 34000, 28500)` image at the `(16, 1024, 1024)` label chunk cap
+    is 7,616 files per label group, well under the 200,000 at which the
+    conversion starts warning. On scicore that is fine; on a filesystem with
+    a tighter inode or per-directory budget it may not be.
+
+!!! tip "Which OME-ZARR version gets written (`ngff_version`)"
+    OME-ZARR is versioned, and the version decides the **zarr format** as
+    well as the metadata layout — the two are not separate choices. NGFF 0.4
+    is specified over zarr v2 and puts `multiscales`, `labels` and
+    `image-label` at the top level of the store's attributes; 0.5 is the
+    zarr-v3 revision and nests them under an `ome` key.
+
+    `ngff_version: "auto"` (the default) follows the installed zarr, which on
+    any current environment means **0.5**. Pin `"0.4"` only when a downstream
+    tool still cannot read 0.5:
+
+    ```yaml
+    ngff_version: "0.4"
+    shard: false           # required: zarr v2 has no sharding codec
+    ```
+
+    That is a real trade-off, not a formality — 0.4 means a zarr-v2 store, and
+    sharding is a zarr-v3 feature, so `shard`/`shard_labels` stop doing
+    anything. The workflow refuses the combination rather than silently
+    ignoring it. The store's own root file changes too (`.zgroup` instead of
+    `zarr.json`), which the rules account for.
+
+    Writing into an **existing** store always follows that store's format,
+    whatever this key says: the label pyramid is written by `merge`, long
+    after `convert` made the image, and a store with v2 arrays and v3
+    metadata is one no reader can open.
+
+    **What about 0.6?** It was released on 14 September 2026 and patchworks
+    does not write it. It is not a version bump but a different metadata
+    document: RFC-5 replaces a multiscale's `axes` with `coordinateSystems`
+    and requires an `input`/`output` pair on every coordinate transformation.
+    Nothing reads it yet either — ome-zarr-py, and so napari, still default
+    to 0.5. Setting `ngff_version: "0.6"` is rejected with that explanation
+    rather than writing a store you could not open.
+
 !!! tip "Dropping objects by size with `min_volume`/`max_volume`"
     `min_volume: N` drops any object smaller than `N` µm³; `max_volume: N`
     drops any object larger than `N` µm³ (e.g. several objects merged into
@@ -112,15 +187,60 @@ sequential_labels: true        # renumber labels to a contiguous 1..N
     [Filtering by size after merge](merging.md#filtering-by-size-after-merge)
     for the equivalent direct-API call.
 
+!!! warning "`model:` names are version-specific — check which Cellpose you have"
+    Cellpose 3 and 4 have **disjoint** model names: v3 has `cyto3`,
+    `nuclei`, …; v4 replaced them all with the `cpsam` family (`cpsam`,
+    `cpsam_v2`, `cpdino`, …). Neither version *raises* on a name it doesn't
+    know — v4 logs a warning and quietly loads its default (`cpsam_v2`)
+    instead. A `cyto3` left in a config against a v4 install therefore
+    segments every tile with a model you did not choose, and the only
+    evidence is one line in the job log.
+
+    patchworks now rejects an unavailable name in `prepare`, on the cheap
+    CPU job, rather than letting it through. Check what you have with
+    `python -c "import cellpose; print(cellpose.version)"`, then either pick
+    a name that install offers, point `model:` at a custom-trained model's
+    path, or install the version you want — the workflow ships `cellpose3`
+    and `cellpose4` pixi environments (`pixi install -e cellpose3`) for
+    exactly this.
+
 !!! tip "3-D anisotropy is derived automatically"
     Cellpose's `do_3D` assumes isotropic voxels unless told otherwise —
     without an `anisotropy`, a real (anisotropic) dataset gets objects
-    fragmented or distorted across z. `segment` now derives it from
+    fragmented or distorted across z. `segment` derives it from
     `image.zarr`'s own calibration (`z` voxel size ÷ lateral voxel size)
     whenever `do_3D: true` and `cellpose.anisotropy` isn't set explicitly, so
     there's usually nothing to configure. Set `anisotropy:` yourself in the
     `cellpose:` block to override it.
 
+    Note it is not just a hint to the model: Cellpose *resizes* the tile to
+    `z × anisotropy` planes before the net runs. `tile_shape: "auto"`
+    budgets for that resized tile, so a 2.2× anisotropy buys a
+    correspondingly smaller tile rather than an out-of-memory job.
+
+!!! warning "The physically-correct anisotropy is not always the best one"
+    Upsampling z by the true ratio can produce **ring artifacts** and costs
+    runtime proportional to the ratio. A contributor on
+    [cellpose#1408](https://github.com/MouseLand/cellpose/pull/1408) reports
+    that `anisotropy: 1` avoids the rings entirely and is much faster, at
+    the cost of boundaries being off by a few pixels in the top/bottom
+    z-planes, where a cell's cross-section changes fastest — and recommends
+    pairing it with light z-only flow smoothing:
+
+    ```yaml
+    cellpose:
+      do_3D: true
+      anisotropy: 1          # overrides the derived value
+      flow3D_smooth: [1, 0, 0]   # z, y, x -- smooth z only
+    ```
+
+    This is a genuine trade-off, not a strictly better setting, and it is
+    worth testing both on your own data rather than taking either on faith.
+    Reach for it especially if 3-D results look ringed or fragmented along
+    z: that is the symptom this addresses. `flow3D_smooth` accepts a scalar
+    on older Cellpose and a `[z, y, x]` list from the version that merged
+    that PR onward.
+
 !!! tip "Tile size vs runtime"
     `tile_shape: "auto"` sizes each tile to your GPU's VRAM. Smaller tiles =
     more (faster) jobs; very large 3-D tiles are slow. Keep `do_3D: false` (2-D
@@ -317,6 +437,7 @@ input: "/data/scan.ims"
 work_dir: "/scratch/results"
 tile_shape: [16, 1024, 1024]
 shard: false                   # true → far fewer files, same chunks
+ngff_version: "auto"           # convert reads it, so it belongs here
 tiles_per_job: 4
 ```
 
@@ -402,11 +523,21 @@ snakemake --workflow-profile profile/slurm --configfile config/common.yaml confi
 ```
 
 !!! warning "Conversion settings belong in the shared file"
-    `convert` runs **once**, from the first config only. A `shard`, `input` or
-    `pyramid_levels` set on the second config is therefore never read, and
-    nothing logs that it was dropped. `run_multi` refuses to start when those
-    keys disagree across configs and tells you which one — but if you drive
-    the configs by hand, keep them in `common.yaml`.
+    `convert` runs **once**, from the first config only. A `shard`, `input`,
+    `convert_chunks`, `sequence_pattern` or `reuse_pyramid` set on the second
+    config is therefore never read, and nothing logs that it was dropped.
+    `run_multi` refuses to start when those keys disagree across configs and
+    tells you which one — but if you drive the configs by hand, keep them in
+    `common.yaml`.
+
+    `ngff_version` is in that list: it decides the store's zarr format, so a
+    second config disagreeing about it would describe a store that is not the
+    one on disk.
+
+    `merge` runs once **per config**, so the keys it reads —
+    `pyramid_levels`, `pyramid_downscale`, `sequential_labels`,
+    `min_volume`/`max_volume` and `shard_labels` — may legitimately differ
+    between them, and are only in `common.yaml` for convenience.
 
     Splitting the configs is optional: a self-contained config still works,
     and `common:` can simply be left out of `multi.yaml`.

{patchworks-2.6.11 → patchworks-2.7.0}/pyproject.toml

@@ -96,7 +96,16 @@ napari = [
 # openpyxl -> scripts/run_multi.py writes label_relations() output as an
 # Excel workbook (per-object + per-container sheets), not a plain CSV.
 workflow = ["snakemake>=8", "snakemake-executor-plugin-slurm", "openpyxl"]
-dev = ["pytest", "pytest-cov", "scikit-image", "psutil", "tqdm"]
+# jsonschema validates what we write against the vendored official
+# OME-NGFF schemas (tests/ngff_schemas/); without it that one test skips.
+dev = [
+    "pytest",
+    "pytest-cov",
+    "scikit-image",
+    "psutil",
+    "tqdm",
+    "jsonschema>=4.18",
+]
 docs = ["mkdocs-material>=9.0", "mkdocstrings[python]>=0.24"]
 all = [
     "patchworks[io,gpu,bioio,imaris,napari]",

{patchworks-2.6.11 → patchworks-2.7.0}/src/patchworks/_chunks.py

@@ -380,6 +380,7 @@ def auto_tile_shape_cellpose(
     model_memory_bytes: int = 2 * 1024**3,
     cellpose_memory_factor: int = 20,
     n_channels: int = 1,
+    anisotropy: float | None = None,
     verbose: bool = False,
 ) -> tuple[int, ...]:
     """Cellpose-optimised tile shape.
@@ -426,6 +427,13 @@ def auto_tile_shape_cellpose(
         Channels each tile carries (default 1). Above 1 the per-voxel cost
         scales with it, so the tile shrinks accordingly -- e.g. the workflow's
         ``nuclei_channel`` hands Cellpose a cyto+nuclei pair.
+    anisotropy:
+        The ``anisotropy`` Cellpose will be given (``do_3D`` only). It is not
+        merely a hint to the model: Cellpose resizes the tile to
+        ``z * anisotropy`` planes before the net runs, so a 2.2x anisotropy
+        costs 2.2x the z it was handed. Budgeting against the unscaled ``z``
+        under-counts by exactly that factor. ``None`` (the default) assumes
+        isotropic, i.e. no resize.
     verbose:
         Log the chosen shape and memory estimates.
 
@@ -441,6 +449,18 @@ def auto_tile_shape_cellpose(
     (1, 2048, 2048)
     """
     n_workers = n_workers or cpu_allocation()
+    # Cellpose resizes the tile before the net runs -- by `rescale`
+    # (= 30 / diameter) on every axis, and by `anisotropy` on z as well -- so
+    # the array it actually holds is bigger than the one it was handed, and a
+    # budget computed from the unresized tile under-counts by that factor. A
+    # diameter half the model's 30 px means a 2x upsample per axis: 8x the
+    # voxels, enough to turn a comfortable tile into an OOM.
+    #
+    # Both only ever *shrink* the tile. A predicted downsample (diameter > 30)
+    # would license a bigger one, but these are a safety margin against a
+    # rough memory model, not a measurement to spend headroom on.
+    rescale = max(1.0, 30.0 / diameter) if diameter else 1.0
+    z_resize = max(1.0, anisotropy or 1.0)
     # A tile holds n_channels planes per voxel (e.g. Cellpose's
     # cyto+nuclei pair), so the per-voxel cost -- and every budget
     # derived from it below -- scales with them.
@@ -467,7 +487,8 @@ def auto_tile_shape_cellpose(
     min_tile = int(4 * diameter) if diameter is not None else 1
 
     if n_spatial == 2 or not do_3D:
-        max_pixels_2d = max(1, max_raw_bytes // itemsize)
+        # Two axes resized, so the cost per configured pixel is rescale**2.
+        max_pixels_2d = max(1, int(max_raw_bytes // (itemsize * rescale**2)))
         tile_side = max(min_tile, int(max_pixels_2d**0.5))
         if n_spatial == 2:
             y, x = shape[-2], shape[-1]
@@ -477,7 +498,13 @@ def auto_tile_shape_cellpose(
             chunk_spatial = [1, min(y, tile_side), min(x, tile_side)]
     else:
         z, y, x = shape[-3], shape[-2], shape[-1]
-        max_pixels_per_slice = max(1, (max_raw_bytes // 3) // (z * itemsize))
+        # All three axes are resized: z by anisotropy * rescale, y/x by
+        # rescale each -- so one configured voxel costs
+        # anisotropy * rescale**3 of them.
+        effective_z = z * z_resize * rescale**3
+        max_pixels_per_slice = max(
+            1, int((max_raw_bytes // 3) // (effective_z * itemsize))
+        )
         tile_side = max(min_tile, int(max_pixels_per_slice**0.5))
         chunk_spatial = [z, min(y, tile_side), min(x, tile_side)]
 

{patchworks-2.6.11 → patchworks-2.7.0}/src/patchworks/_io.py

@@ -18,27 +18,38 @@ _ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
 _ZARR_V3 = int(zarr.__version__.split(".")[0]) >= 3
 
 
-def zarr_compressor_kwargs() -> dict:
+def zarr_compressor_kwargs(zarr_format: int = 3) -> dict:
     """Keyword arguments pinning the compression codec for a new array.
 
     zstd is already zarr v3's default, but relying on a library default means
     the stores patchworks writes change silently if that default ever moves.
     Labels in particular are highly compressible, so this is worth stating.
 
+    The codec *object* depends on the format of the array being written, not
+    on the installed zarr: zarr-python 3 can write a zarr-v2 array (which is
+    what NGFF 0.4 needs), and a v2 array rejects ``zarr.codecs.ZstdCodec`` --
+    it wants the numcodecs one.
+
+    Parameters
+    ----------
+    zarr_format : int, optional
+        Format of the array about to be created, 2 or 3 (default 3).
+
     Returns
     -------
     dict
-        ``compressors=``/``compressor=`` as the installed zarr expects, or
+        ``compressors=``/``compressor=`` as that combination expects, or
         empty if the codec cannot be built (then the default applies).
     """
     try:
-        if _ZARR_V3:
+        if _ZARR_V3 and zarr_format != 2:
             from zarr.codecs import ZstdCodec
 
             return {"compressors": (ZstdCodec(level=1),)}
         import numcodecs
 
-        return {"compressor": numcodecs.Zstd(level=1)}
+        codec = numcodecs.Zstd(level=1)
+        return {"compressors": (codec,)} if _ZARR_V3 else {"compressor": codec}
     except Exception:  # pragma: no cover - depends on the installed zarr
         logger.debug("could not pin a compressor; using zarr's default")
         return {}

{patchworks-2.6.11 → patchworks-2.7.0}/src/patchworks/plugins/cellpose.py

@@ -253,6 +253,26 @@ def _make_config(
     }
 
 
+def available_models() -> list[str]:
+    """Pretrained model names the installed Cellpose actually accepts.
+
+    The name set changed completely between major versions -- v3 ships
+    ``cyto3``/``nuclei``/… , v4 replaced them with ``cpsam``-family names --
+    and neither version raises on an unknown one. v4 falls back to its
+    default model with only a log line, so a v3 name in a v4 environment
+    segments everything with the wrong model and nothing fails. Checking the
+    name against this list turns that into a config error instead.
+
+    Returns
+    -------
+    list of str
+        Model names, or an empty list when Cellpose isn't installed or
+        doesn't publish them (in which case no name can be rejected).
+    """
+    names = getattr(_cellpose_models, "MODEL_NAMES", None)
+    return list(names) if names else []
+
+
 def _get_model(cellpose_dict: dict[str, Any]) -> Any:
     """Return a worker-local cached Cellpose model.
 
@@ -272,8 +292,14 @@ def _get_model(cellpose_dict: dict[str, Any]) -> Any:
         gpu = cellpose_dict.get("gpu", False)
         model_type = cellpose_dict["model"]
         if _CELLPOSE_V4:
+            # v4 renamed this: `model_type=` is accepted but explicitly
+            # ignored ("not used in v4.0.1+"), leaving pretrained_model at
+            # its "cpsam_v2" default -- so passing the configured name there
+            # silently segmented *every* config with the same default model,
+            # whatever `model:` said, with only a logger warning to show for
+            # it. `available_models()` rejects an unusable name up front.
             _model_cache[key] = _cellpose_models.CellposeModel(
-                model_type=model_type, gpu=gpu
+                pretrained_model=model_type, gpu=gpu
             )
         else:
             _model_cache[key] = _cellpose_models.Cellpose(