patchworks 0.11.15__tar.gz → 0.11.16__tar.gz

{patchworks-0.11.15 → patchworks-0.11.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 0.11.15
+Version: 0.11.16
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-0.11.15 → patchworks-0.11.16}/workflow/config/config.yaml

@@ -51,3 +51,7 @@ cellpose:
 pyramid_levels: 5
 pyramid_downscale: 2
 sequential_labels: true # renumber merged labels to a contiguous 1..N range
+
+# ---- merge performance --------------------------------------------------
+# null -> use the "merge" rule's actual cpus_per_task (profile/slurm/config.yaml).
+merge_workers: null

{patchworks-0.11.15 → patchworks-0.11.16}/workflow/profile/slurm/config.yaml

@@ -5,7 +5,13 @@
 # Edit the partitions / account / GPU request for your cluster.
 
 executor: slurm
-jobs: 64 # max concurrent SLURM jobs (≈ how many GPUs you can grab at once)
+# Max concurrent SLURM jobs Snakemake will have in flight at once. This is a
+# submission-side cap, not a cluster limit — raising it just lets more
+# "segment" tiles queue/run together; scicore's scheduler still enforces your
+# real GPU quota (MaxTRESPU) and backfills the rest, so it's safe to raise.
+# Check your actual per-user GPU cap with:
+#   sacctmgr show assoc user=$USER format=account,partition,maxjobs,GrpTRES
+jobs: 128
 latency-wait: 60
 rerun-incomplete: true
 # Only re-run when an output is missing/stale — NOT when the patchworks code,
@@ -26,8 +32,12 @@ default-resources:
 # Per-rule overrides.
 set-resources:
   convert:
+    # bioio/dask's default threaded scheduler uses every visible core, so
+    # this scales conversion for free. Verify actual scaling with `seff
+    # <jobid>` after a run and dial back if efficiency is low (scicore
+    # fair-share asks you not to sit on idle cores).
     mem_mb: 64000
-    cpus_per_task: 8
+    cpus_per_task: 32
     runtime: 360
   prepare:
     mem_mb: 32000
@@ -44,6 +54,8 @@ set-resources:
     cpus_per_task: 4
     runtime: 360 # 6 hours — must match the QOS, NOT 120 (=2h → killed early)
   merge:
+    # merge_tile_labels now uses n_workers = this allocation (see merge.py /
+    # config.yaml's merge_workers) instead of silently capping at 4 threads.
     mem_mb: 128000
-    cpus_per_task: 8
+    cpus_per_task: 32
     runtime: 240

{patchworks-0.11.15 → patchworks-0.11.16}/workflow/scripts/merge.py

@@ -5,6 +5,7 @@ boundaries, optionally renumbers them) and writes the result back into the
 image store under ``labels/<name>/`` as a calibrated, multi-scale pyramid.
 """
 
+import os
 import shutil
 from pathlib import Path
 
@@ -19,11 +20,17 @@ work_dir = cfg["work_dir"]
 image_store = str(Path(work_dir) / "image.zarr")
 merged_store = str(Path(work_dir) / "_merged.zarr")
 
+# merge_tile_labels defaults to min(4, cpu_count) workers, so it ignores
+# whatever cpus_per_task the "merge" rule was actually allocated in the SLURM
+# profile. Read the real allocation (SLURM_CPUS_PER_TASK) so the job uses all
+# the cores it's paying for; merge_workers: in config.yaml can still override.
+default_workers = int(os.environ.get("SLURM_CPUS_PER_TASK", os.cpu_count() or 4))
 merged = merge_tile_labels(
     stage_path(work_dir),
     write_to=merged_store,
     input_component="staged",
     sequential_labels=cfg.get("sequential_labels", True),
+    n_workers=cfg.get("merge_workers", default_workers),
     progress=False,
 )
 group = write_labels(
@@ -37,5 +44,10 @@ group = write_labels(
 
 shutil.rmtree(merged_store, ignore_errors=True)
 shutil.rmtree(stage_path(work_dir), ignore_errors=True)
+# Also drop the checkpoint's completion sentinel (stage.zarr.done): the
+# "prepare" rule's stage=touch(STAGE_OK) output must not outlive the store it
+# claims exists, or a future rerun (e.g. re-segmenting for new labels) skips
+# "prepare" and "segment" tries to open a stage.zarr that's already gone.
+Path(f"{stage_path(work_dir)}.done").unlink(missing_ok=True)
 print(f"[patchworks] labels written to {group}")
 open(snakemake.output[0], "w").close()  # noqa: F821