patchworks 0.11.14__tar.gz → 0.11.16__tar.gz

{patchworks-0.11.14 → patchworks-0.11.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 0.11.14
+Version: 0.11.16
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues
@@ -50,6 +50,10 @@ Requires-Dist: bioio-ome-tiff; extra == 'bioio'
 Requires-Dist: bioio-tifffile; extra == 'bioio'
 Provides-Extra: cellpose
 Requires-Dist: cellpose>=3.0; extra == 'cellpose'
+Provides-Extra: cellpose3
+Requires-Dist: cellpose<4,>=3.0; extra == 'cellpose3'
+Provides-Extra: cellpose4
+Requires-Dist: cellpose>=4; extra == 'cellpose4'
 Provides-Extra: dev
 Requires-Dist: psutil; extra == 'dev'
 Requires-Dist: pytest; extra == 'dev'
@@ -115,12 +119,14 @@ pip install patchworks
 Optional extras:
 
 ```bash
-pip install "patchworks[gpu]"      # GPU VRAM querying (nvidia-ml-py)
-pip install "patchworks[cellpose]" # Cellpose plugin
-pip install "patchworks[bioio]"    # convert any image format to OME-ZARR
-pip install "patchworks[imaris]"   # convert Imaris .ims files to OME-ZARR
-pip install "patchworks[napari]"   # interactive napari viewer plugin
-pip install "patchworks[all]"      # Everything, incl. the napari viewer
+pip install "patchworks[gpu]"       # GPU VRAM querying (nvidia-ml-py)
+pip install "patchworks[cellpose]"  # Cellpose plugin (>=3.0, v3 or v4)
+pip install "patchworks[cellpose3]" # Cellpose plugin, pinned to v3.x
+pip install "patchworks[cellpose4]" # Cellpose plugin, pinned to v4+
+pip install "patchworks[bioio]"     # convert any image format to OME-ZARR
+pip install "patchworks[imaris]"    # convert Imaris .ims files to OME-ZARR
+pip install "patchworks[napari]"    # interactive napari viewer plugin
+pip install "patchworks[all]"       # Everything, incl. the napari viewer
 ```
 
 > `bioio` reads CZI/LIF/ND2/OME-TIFF/… The `[bioio]` extra bundles the common
@@ -382,7 +388,8 @@ Optional:
 - `psutil` — accurate RAM sizing for `tile_shape="auto"`
 - `nvidia-ml-py` — accurate GPU VRAM sizing
 - `tqdm` — progress bars
-- `cellpose` — Cellpose plugin (`patchworks[cellpose]`)
+- `cellpose` — Cellpose plugin, v3 or v4 (`patchworks[cellpose]`);
+  pin with `[cellpose3]` or `[cellpose4]`
 - `bioio` + readers — convert CZI/LIF/ND2/OME-TIFF/… to OME-ZARR
   (`patchworks[bioio]`)
 - `imaris-ims-file-reader` — convert Imaris `.ims` (`patchworks[imaris]`)

{patchworks-0.11.14 → patchworks-0.11.16}/README.md

@@ -36,12 +36,14 @@ pip install patchworks
 Optional extras:
 
 ```bash
-pip install "patchworks[gpu]"      # GPU VRAM querying (nvidia-ml-py)
-pip install "patchworks[cellpose]" # Cellpose plugin
-pip install "patchworks[bioio]"    # convert any image format to OME-ZARR
-pip install "patchworks[imaris]"   # convert Imaris .ims files to OME-ZARR
-pip install "patchworks[napari]"   # interactive napari viewer plugin
-pip install "patchworks[all]"      # Everything, incl. the napari viewer
+pip install "patchworks[gpu]"       # GPU VRAM querying (nvidia-ml-py)
+pip install "patchworks[cellpose]"  # Cellpose plugin (>=3.0, v3 or v4)
+pip install "patchworks[cellpose3]" # Cellpose plugin, pinned to v3.x
+pip install "patchworks[cellpose4]" # Cellpose plugin, pinned to v4+
+pip install "patchworks[bioio]"     # convert any image format to OME-ZARR
+pip install "patchworks[imaris]"    # convert Imaris .ims files to OME-ZARR
+pip install "patchworks[napari]"    # interactive napari viewer plugin
+pip install "patchworks[all]"       # Everything, incl. the napari viewer
 ```
 
 > `bioio` reads CZI/LIF/ND2/OME-TIFF/… The `[bioio]` extra bundles the common
@@ -303,7 +305,8 @@ Optional:
 - `psutil` — accurate RAM sizing for `tile_shape="auto"`
 - `nvidia-ml-py` — accurate GPU VRAM sizing
 - `tqdm` — progress bars
-- `cellpose` — Cellpose plugin (`patchworks[cellpose]`)
+- `cellpose` — Cellpose plugin, v3 or v4 (`patchworks[cellpose]`);
+  pin with `[cellpose3]` or `[cellpose4]`
 - `bioio` + readers — convert CZI/LIF/ND2/OME-TIFF/… to OME-ZARR
   (`patchworks[bioio]`)
 - `imaris-ims-file-reader` — convert Imaris `.ims` (`patchworks[imaris]`)

{patchworks-0.11.14 → patchworks-0.11.16}/pyproject.toml

@@ -41,6 +41,8 @@ dependencies = [
 [project.optional-dependencies]
 io = ["psutil", "tqdm"]
 cellpose = ["cellpose>=3.0"]
+cellpose3 = ["cellpose>=3.0,<4"]
+cellpose4 = ["cellpose>=4"]
 gpu = ["nvidia-ml-py"]
 # bioio enables converting any image format (CZI, LIF, ND2, OME-TIFF, …) to
 # OME-ZARR. Ships the common native readers (faster than going through Java)

{patchworks-0.11.14 → patchworks-0.11.16}/workflow/README.md

@@ -41,10 +41,17 @@ pip install "patchworks[workflow,cellpose,imaris,bioio]"
 Prefer pixi? No conda needed — a `pixi.toml` is included:
 
 ```bash
-pixi install
-pixi run dry    # dry-run
-pixi run go     # run locally
-pixi run slurm  # submit to SLURM
+pixi install               # default env (latest compatible Cellpose)
+pixi install -e cellpose3  # pin to Cellpose 3.x
+pixi install -e cellpose4  # pin to Cellpose 4+
+
+pixi run dry               # dry-run (default env)
+pixi run go                # run locally (default env)
+pixi run slurm             # submit to SLURM (default env)
+
+# To run with a specific Cellpose version:
+pixi run -e cellpose3 go
+pixi run -e cellpose4 go
 ```
 
 ## Configure

{patchworks-0.11.14 → patchworks-0.11.16}/workflow/config/config.yaml

@@ -51,3 +51,7 @@ cellpose:
 pyramid_levels: 5
 pyramid_downscale: 2
 sequential_labels: true # renumber merged labels to a contiguous 1..N range
+
+# ---- merge performance --------------------------------------------------
+# null -> use the "merge" rule's actual cpus_per_task (profile/slurm/config.yaml).
+merge_workers: null

{patchworks-0.11.14 → patchworks-0.11.16}/workflow/pixi.toml

@@ -15,9 +15,23 @@ platforms = ["linux-64"]
 [dependencies]
 python = "3.12.*"
 
+# Default env uses patchworks[cellpose] (>=3.0, resolves to latest compatible).
 [pypi-dependencies]
 patchworks = { version = "*", extras = ["workflow", "cellpose", "imaris", "bioio"] }
 
+# Cellpose-version-pinned features for explicit environments below.
+[feature.cp3.pypi-dependencies]
+cellpose = ">=3.0,<4"
+
+[feature.cp4.pypi-dependencies]
+cellpose = ">=4"
+
+# Named environments: pixi install -e cellpose3 / -e cellpose4
+# The base patchworks[cellpose] (>=3.0) is further constrained by each feature.
+[environments]
+cellpose3 = { features = ["cp3"], solve-group = "cp3" }
+cellpose4 = { features = ["cp4"], solve-group = "cp4" }
+
 [tasks]
 # --rerun-triggers mtime: don't redo finished steps (e.g. the conversion) just
 # because the patchworks version/code changed — only when an output is missing.

{patchworks-0.11.14 → patchworks-0.11.16}/workflow/profile/slurm/config.yaml

@@ -5,7 +5,13 @@
 # Edit the partitions / account / GPU request for your cluster.
 
 executor: slurm
-jobs: 64 # max concurrent SLURM jobs (≈ how many GPUs you can grab at once)
+# Max concurrent SLURM jobs Snakemake will have in flight at once. This is a
+# submission-side cap, not a cluster limit — raising it just lets more
+# "segment" tiles queue/run together; scicore's scheduler still enforces your
+# real GPU quota (MaxTRESPU) and backfills the rest, so it's safe to raise.
+# Check your actual per-user GPU cap with:
+#   sacctmgr show assoc user=$USER format=account,partition,maxjobs,GrpTRES
+jobs: 128
 latency-wait: 60
 rerun-incomplete: true
 # Only re-run when an output is missing/stale — NOT when the patchworks code,
@@ -26,8 +32,12 @@ default-resources:
 # Per-rule overrides.
 set-resources:
   convert:
+    # bioio/dask's default threaded scheduler uses every visible core, so
+    # this scales conversion for free. Verify actual scaling with `seff
+    # <jobid>` after a run and dial back if efficiency is low (scicore
+    # fair-share asks you not to sit on idle cores).
     mem_mb: 64000
-    cpus_per_task: 8
+    cpus_per_task: 32
     runtime: 360
   prepare:
     mem_mb: 32000
@@ -44,6 +54,8 @@ set-resources:
     cpus_per_task: 4
     runtime: 360 # 6 hours — must match the QOS, NOT 120 (=2h → killed early)
   merge:
+    # merge_tile_labels now uses n_workers = this allocation (see merge.py /
+    # config.yaml's merge_workers) instead of silently capping at 4 threads.
     mem_mb: 128000
-    cpus_per_task: 8
+    cpus_per_task: 32
     runtime: 240

{patchworks-0.11.14 → patchworks-0.11.16}/workflow/scripts/merge.py

@@ -5,6 +5,7 @@ boundaries, optionally renumbers them) and writes the result back into the
 image store under ``labels/<name>/`` as a calibrated, multi-scale pyramid.
 """
 
+import os
 import shutil
 from pathlib import Path
 
@@ -19,11 +20,17 @@ work_dir = cfg["work_dir"]
 image_store = str(Path(work_dir) / "image.zarr")
 merged_store = str(Path(work_dir) / "_merged.zarr")
 
+# merge_tile_labels defaults to min(4, cpu_count) workers, so it ignores
+# whatever cpus_per_task the "merge" rule was actually allocated in the SLURM
+# profile. Read the real allocation (SLURM_CPUS_PER_TASK) so the job uses all
+# the cores it's paying for; merge_workers: in config.yaml can still override.
+default_workers = int(os.environ.get("SLURM_CPUS_PER_TASK", os.cpu_count() or 4))
 merged = merge_tile_labels(
     stage_path(work_dir),
     write_to=merged_store,
     input_component="staged",
     sequential_labels=cfg.get("sequential_labels", True),
+    n_workers=cfg.get("merge_workers", default_workers),
     progress=False,
 )
 group = write_labels(
@@ -37,5 +44,10 @@ group = write_labels(
 
 shutil.rmtree(merged_store, ignore_errors=True)
 shutil.rmtree(stage_path(work_dir), ignore_errors=True)
+# Also drop the checkpoint's completion sentinel (stage.zarr.done): the
+# "prepare" rule's stage=touch(STAGE_OK) output must not outlive the store it
+# claims exists, or a future rerun (e.g. re-segmenting for new labels) skips
+# "prepare" and "segment" tries to open a stage.zarr that's already gone.
+Path(f"{stage_path(work_dir)}.done").unlink(missing_ok=True)
 print(f"[patchworks] labels written to {group}")
 open(snakemake.output[0], "w").close()  # noqa: F821