patchworks 2.6.2__tar.gz → 2.6.4__tar.gz

{patchworks-2.6.2 → patchworks-2.6.4}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: patchworks
-Version: 2.6.2
+Version: 2.6.4
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-2.6.2 → patchworks-2.6.4}/docs/guide/snakemake.md

@@ -451,6 +451,18 @@ the GPU partition stays busy instead of idling through every config's
 `prepare` and multi-hour `merge` in turn. A config that fails does **not**
 abort the others; you get a per-config status and a non-zero exit.
 
+!!! tip "The relate step runs on the cluster too, under `multi-slurm`"
+    `label_relations()` streams every chunk of two full-resolution label
+    volumes — real CPU/IO work, not orchestration. Under `multi-slurm` it is
+    submitted as its own `srun` job (`scripts/relate.py`) instead of running
+    in the driver process on the login node, the same fix already applied to
+    the occupancy map. Tune its allocation with `--relate-partition`,
+    `--relate-mem`, `--relate-cpus` and `--relate-time` (defaults: `scicore`,
+    `32G`, `8`, `180` minutes) — these are wide-margin guesses, not measured
+    numbers, so raise them for a very large or very object-dense pair. Under
+    plain `multi` (no `--profile`), it still runs locally, in-process, as
+    before.
+
 !!! tip "After a killed run"
     Snakemake only releases its lock on a clean exit, so a run that was killed
     (Ctrl-C, an SSH drop, an OOM) leaves the directory locked. Each phase has

{patchworks-2.6.2 → patchworks-2.6.4}/src/patchworks/_notify.py

@@ -159,7 +159,7 @@ def slurm_mail_extra(
 
 
 def failing_step(
-    snakemake_log: Union[str, Path, None],
+    snakemake_log: "Union[str, Path, list, tuple, None]",
 ) -> "tuple[Union[str, None], Union[str, None]]":
     """Find which rule failed, and its log, from Snakemake's own log file.
 
@@ -171,9 +171,12 @@ def failing_step(
 
     Parameters
     ----------
-    snakemake_log : str or Path or None
-        Path to Snakemake's own log (the ``log`` variable inside an
-        ``onerror`` handler).
+    snakemake_log : str, Path, list, tuple, or None
+        Snakemake's own log (the ``log`` variable inside an ``onerror``
+        handler). Snakemake 8+ passes this as a list of paths (its
+        ``LoggerManager.get_logfile()`` returns ``List[str]``, even though
+        there is normally just one) rather than a single string -- take the
+        first entry.
 
     Returns
     -------
@@ -181,6 +184,8 @@ def failing_step(
         ``(rule_name, log_path)``, either of which may be None when the log
         is unreadable or records no rule error.
     """
+    if isinstance(snakemake_log, (list, tuple)):
+        snakemake_log = snakemake_log[0] if snakemake_log else None
     try:
         text = Path(snakemake_log).read_text(errors="replace")
     except (OSError, TypeError, ValueError):

{patchworks-2.6.2 → patchworks-2.6.4}/tests/test_notify.py

@@ -99,6 +99,30 @@ def test_failing_step_reads_the_rule_from_snakemakes_log(tmp_path):
     )
 
 
+def test_failing_step_accepts_snakemakes_list_log(tmp_path):
+    """Snakemake 8+ passes `log` as a list, not a bare path.
+
+    ``LoggerManager.get_logfile()`` returns ``List[str]``, and that's what
+    the `log` variable inside `onerror:` actually is. `Path(a_list)` raises
+    `TypeError`, which the old code caught and turned into `(None, None)` --
+    so this path was *always* silently falling back to the mtime guess it
+    was written to replace, on every real failure.
+    """
+    from patchworks._notify import failing_step
+
+    log = tmp_path / "sm.log"
+    log.write_text(
+        "Error in rule segment:\n"
+        "    jobid: 69\n"
+        "    output: seg/61.done\n"
+        "    log: /w/nuclei_labels/logs/segment/61.log (check log file(s))\n"
+    )
+    expected = ("segment", "/w/nuclei_labels/logs/segment/61.log")
+    assert failing_step([str(log)]) == expected
+    assert failing_step((str(log),)) == expected
+    assert failing_step([]) == (None, None)
+
+
 def test_failing_step_degrades_quietly(tmp_path):
     """It runs inside an error handler, so it must never raise itself."""
     from patchworks._notify import failing_step

{patchworks-2.6.2 → patchworks-2.6.4}/tests/test_run_multi.py

@@ -160,6 +160,32 @@ def test_occupancy_is_not_rebuilt_by_the_driver():
     assert "occupancy.zarr" in src
 
 
+def test_relate_is_submitted_via_slurm_under_profile():
+    """The relate step must never run in-process on the submit host.
+
+    Same failure mode as the occupancy map: label_relations() streams every
+    chunk of two full-resolution label volumes. A prior fix moved the map
+    build off the login node; the relate step made the identical mistake and
+    hung there for the same reason until this fix.
+    """
+    src = (_workflow_dir() / "scripts" / "run_multi.py").read_text()
+    assert "from relate import run_relations" in src
+    assert '"srun"' in src
+    assert "label_relations(" not in src
+
+
+def test_relate_script_has_the_real_bookkeeping():
+    """relate.py must be the actual implementation, not a stub.
+
+    Submitting the wrong (or a trimmed-down) script would silently produce a
+    workbook missing the unmatched-label rows the docstring promises.
+    """
+    src = (_workflow_dir() / "scripts" / "relate.py").read_text()
+    assert "def run_relations(" in src
+    assert "label_relations" in src
+    assert "openpyxl" in src
+
+
 def test_auto_tile_shape_with_a_lone_nuclei_channel_is_refused():
     """Matching `tile_shape` *values* are not enough when one config is 2-ch.
 

{patchworks-2.6.2 → patchworks-2.6.4}/workflow/profile/slurm/config.yaml

@@ -74,7 +74,7 @@ set-resources:
     # equal to segment's mem_mb below: lower and tiles come out needlessly
     # small; higher and the sizer budgets against more host RAM than segment
     # will actually get, undoing the point of the host-RAM check.
-    mem_mb: "attempt * 32000"
+    mem_mb: "attempt * 128000"
     runtime: 120
   segment:
     # one GPU per job — this is what spreads Cellpose across GPUs.
@@ -83,15 +83,35 @@ set-resources:
     # leaves the job without a device.)
     slurm_partition: "rtx4090"
     gres: "gpu:1"
-    qos: "rtx4090-6hours" # scicore: <partition>-<duration> QOS
+    # rtx4090-6hours wasn't enough: a do_3D whole-z tile (126 planes, full
+    # footprint) can run past 6h even though most same-sized tiles finish in
+    # 2.5-3.5h -- retrying it bought nothing since `runtime` didn't scale with
+    # `attempt` the way `mem_mb` does below, so it hit the identical wall on
+    # every attempt and burned all `retries` for a job that was never going to
+    # finish in the window. rtx4090-1day is scicore's next tier up (see
+    # `sacctmgr show qos format=name,maxwall | grep rtx4090`); 24h is wide
+    # margin over anything observed so far. If a tile still times out at 24h,
+    # that's not a slow tile anymore -- it's stuck, and worth profiling on its
+    # own rather than reaching for rtx4090-1week.
+    qos: "rtx4090-1day" # scicore: <partition>-<duration> QOS
     # A job now processes `tiles_per_job` tiles sequentially, so both memory
     # and runtime scale with that setting — raise it there and re-check here.
     # The old "a tile used ~1G" note predates tile_shape: "auto", which sizes
     # tiles against the real GPU and makes them far bigger. If you raise
     # this, raise prepare's mem_mb above to match (see its comment).
-    mem_mb: "attempt * 32000"
+    #
+    # 128 GB base (256/384 GB on retry) is a wide safety margin, not a tight
+    # estimate: a do_3D tile that `auto_tile_shape_cellpose`'s 20x
+    # cellpose_memory_factor judged safe within a 24 GiB GPU budget still hit
+    # a real 32 GB host OOM, so that heuristic underestimates do_3D's actual
+    # host-RAM use by more than expected. rtx4090 nodes have ~1 TB RAM
+    # (~800 GB usable) and the QOS caps at 4 TB account-wide, so this has
+    # plenty of room -- it buys time until do_3D gets a properly measured
+    # memory factor (e.g. from `seff` on a job that completes) instead of a
+    # guess. ``ponytail:`` tighten this once real peak-RSS numbers exist.
+    mem_mb: "attempt * 128000"
     cpus_per_task: 4
-    runtime: 360 # 6 hours — must match the QOS, NOT 120 (=2h → killed early)
+    runtime: 1400 # ~23.3h — must stay under the rtx4090-1day QOS cap (24h)
   merge:
     # merge.py sizes its worker pool from the cgroup/SLURM budget rather than
     # the node's core count, and the pyramid is written zarr-natively (one

patchworks-2.6.4/workflow/scripts/relate.py

@@ -0,0 +1,149 @@
+"""Compute label_relations for configured pairs and write .xlsx.
+
+Split out of run_multi.py so this step can be submitted as its own SLURM job
+instead of running in-process on the login node. It streams every chunk of
+two full-resolution label volumes -- real CPU/IO work, not orchestration --
+same reasoning as the occupancy-map fix (see run_multi.py's phase A comment).
+
+Usage (called by run_multi.py under --profile, but also runnable standalone,
+e.g. under srun):
+    python scripts/relate.py --work-dir /path/to/work_dir \
+        --image-store /path/to/work_dir/image.zarr \
+        --relations '[{"a": "nuclei_labels", "b": "cyto_labels", "output": "nuclei_to_cyto.xlsx"}]'
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def _label_ids(image_store: str, name: str) -> list[int]:
+    """Ids present in a label image, without scanning the volume.
+
+    The merge writes n_objects/sequential_labels into the label group's
+    attrs precisely so consumers don't have to re-derive the id set; the ids
+    are 1..n_objects by construction. Fall back to a full scan only for a
+    label group written before those attrs existed.
+    """
+    import dask.array as da
+    import zarr
+
+    attrs = dict(zarr.open_group(f"{image_store}/labels/{name}").attrs)
+    if attrs.get("sequential_labels") and attrs.get("n_objects") is not None:
+        return list(range(1, int(attrs["n_objects"]) + 1))
+    print(
+        f"[relate] {name}: no n_objects attr, falling back to a full scan "
+        "for its id set",
+        flush=True,
+    )
+    arr = da.from_zarr(image_store, component=f"labels/{name}/0")
+    return sorted(int(x) for x in da.unique(arr[arr > 0]).compute())
+
+
+def run_relations(
+    work_dir: str, image_store: str, relations: list[dict]
+) -> None:
+    """Compute and write every configured relation pair as an .xlsx workbook.
+
+    Parameters
+    ----------
+    work_dir : str
+        Directory relation workbooks are written into (a relation's
+        ``output``, when relative, resolves against this).
+    image_store : str
+        The shared ``image.zarr`` holding every config's ``labels/<name>``.
+    relations : list of dict
+        Each ``{"a": ..., "b": ..., "output": ...}`` (``output`` optional,
+        defaults to ``<a>_to_<b>.xlsx``), matching ``multi.yaml``'s
+        ``relations:`` list.
+    """
+    import dask.array as da
+    import openpyxl
+
+    from patchworks import label_relations
+
+    for rel in relations:
+        a_name, b_name = rel["a"], rel["b"]
+        out_path = Path(work_dir) / rel.get(
+            "output", f"{a_name}_to_{b_name}.xlsx"
+        )
+        print(f"[relate] relating {a_name} -> {b_name} …", flush=True)
+        a = da.from_zarr(image_store, component=f"labels/{a_name}/0")
+        b = da.from_zarr(image_store, component=f"labels/{b_name}/0")
+        table = label_relations(a, b)
+
+        # label_relations() only returns a-objects that touch a b-object.
+        # Pull the full id sets so unmatched a-objects (zero overlap) and
+        # b-objects with no matches at all still get a row -- otherwise
+        # they'd silently vanish instead of counting as zero.
+        a_ids = _label_ids(image_store, a_name)
+        b_ids = _label_ids(image_store, b_name)
+
+        per_b = {b_id: {"count": 0, "overlap_voxels": 0} for b_id in b_ids}
+        for m in table.values():
+            agg = per_b.get(m["match"])
+            if agg is not None:
+                agg["count"] += 1
+                agg["overlap_voxels"] += m["overlap_voxels"]
+
+        wb = openpyxl.Workbook()
+        ws_a = wb.active
+        ws_a.title = a_name[:31]  # Excel sheet-name length limit
+        ws_a.append(
+            [
+                f"{a_name}_id",
+                f"{b_name}_id",
+                "overlap_voxels",
+                "overlap_fraction",
+            ]
+        )
+        for a_id in a_ids:
+            m = table.get(a_id)
+            if m is None:
+                ws_a.append([a_id, None, 0, 0])  # no overlap -- still counted
+            else:
+                ws_a.append(
+                    [
+                        a_id,
+                        m["match"],
+                        m["overlap_voxels"],
+                        m["overlap_fraction"],
+                    ]
+                )
+
+        ws_b = wb.create_sheet(title=b_name[:31])
+        ws_b.append(
+            [f"{b_name}_id", f"{a_name}_count", "total_overlap_voxels"]
+        )
+        for b_id in b_ids:
+            agg = per_b[b_id]
+            ws_b.append([b_id, agg["count"], agg["overlap_voxels"]])
+
+        wb.save(out_path)
+        print(
+            f"[relate] wrote {out_path} "
+            f"({len(a_ids)} {a_name}, {len(b_ids)} {b_name})",
+            flush=True,
+        )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--work-dir", required=True)
+    parser.add_argument("--image-store", required=True)
+    parser.add_argument(
+        "--relations",
+        required=True,
+        help=(
+            "JSON list of {a, b, output} dicts, matching multi.yaml's "
+            "relations:"
+        ),
+    )
+    args = parser.parse_args()
+    run_relations(args.work_dir, args.image_store, json.loads(args.relations))
+
+
+if __name__ == "__main__":
+    main()

{patchworks-2.6.2 → patchworks-2.6.4}/workflow/scripts/run_multi.py

@@ -20,12 +20,16 @@ Once all segmentations succeed, each configured relation pair is computed via
 patchworks.label_relations and written as an Excel workbook in work_dir,
 with two sheets: one row per a-object (unmatched ones included, with an
 empty b-id and zeros) and one row per b-object (a-object count + total
-overlap, including b-objects with zero matches).
+overlap, including b-objects with zero matches). Under --profile, this runs
+as a submitted SLURM job (see scripts/relate.py) rather than in-process here
+-- same reasoning as the occupancy-map fix: it streams entire label volumes,
+which is real work, not orchestration, and does not belong on the login node.
 """
 
 from __future__ import annotations
 
 import argparse
+import json
 import re
 import subprocess
 import sys
@@ -337,6 +341,28 @@ def main() -> None:
             "otherwise look identical: no email either way."
         ),
     )
+    parser.add_argument(
+        "--relate-partition",
+        default="scicore",
+        help="SLURM partition for the relate step under --profile (default: scicore)",
+    )
+    parser.add_argument(
+        "--relate-mem",
+        default="32G",
+        help="srun --mem for the relate step under --profile (default: 32G)",
+    )
+    parser.add_argument(
+        "--relate-cpus",
+        type=int,
+        default=8,
+        help="srun --cpus-per-task for the relate step under --profile (default: 8)",
+    )
+    parser.add_argument(
+        "--relate-time",
+        type=int,
+        default=180,
+        help="srun --time in minutes for the relate step under --profile (default: 180)",
+    )
     args = parser.parse_args()
 
     workflow_dir = Path(__file__).resolve().parent.parent
@@ -467,96 +493,47 @@ def main() -> None:
     if args.dry_run or not relations:
         return
 
-    import dask.array as da
-    import openpyxl
-    import zarr
-
-    from patchworks import label_relations
-
-    def _label_ids(name: str) -> list[int]:
-        """Ids present in a label image, without scanning the volume.
-
-        The merge writes n_objects/sequential_labels into the label group's
-        attrs precisely so consumers don't have to re-derive the id set; the
-        ids are 1..n_objects by construction. Fall back to the full scan only
-        for a label group written before those attrs existed -- that scan runs
-        here on the login node, so it is worth avoiding.
-        """
-        attrs = dict(zarr.open_group(f"{image_store}/labels/{name}").attrs)
-        if (
-            attrs.get("sequential_labels")
-            and attrs.get("n_objects") is not None
-        ):
-            return list(range(1, int(attrs["n_objects"]) + 1))
-        print(
-            f"[run_multi] {name}: no n_objects attr, falling back to a full "
-            "scan for its id set",
-            flush=True,
-        )
-        arr = da.from_zarr(image_store, component=f"labels/{name}/0")
-        return sorted(int(x) for x in da.unique(arr[arr > 0]).compute())
+    if args.profile:
+        # Real CPU/IO work -- tens of thousands of zarr chunk reads for a
+        # full-resolution label volume -- not orchestration, so (like the
+        # occupancy map) it does not belong in this driver process on the
+        # login node. Submit it as its own job instead.
+        cmd = [
+            "srun",
+            "--partition",
+            args.relate_partition,
+            "--mem",
+            args.relate_mem,
+            "--cpus-per-task",
+            str(args.relate_cpus),
+            "--time",
+            str(args.relate_time),
+            "--job-name",
+            "pw-relate",
+            sys.executable,
+            str(workflow_dir / "scripts" / "relate.py"),
+            "--work-dir",
+            work_dir,
+            "--image-store",
+            image_store,
+            "--relations",
+            json.dumps(relations),
+        ]
+        rc = _run(cmd, workflow_dir)
+        if rc != 0:
+            print(
+                f"[run_multi] ERROR: relate step failed (exit {rc}). "
+                "Segmentations already succeeded -- only the relation "
+                "workbook(s) are missing. Re-run with the same --config to "
+                "retry just this step.",
+                file=sys.stderr,
+            )
+            sys.exit(rc)
+        return
 
-    for rel in relations:
-        a_name, b_name = rel["a"], rel["b"]
-        out_path = Path(work_dir) / rel.get(
-            "output", f"{a_name}_to_{b_name}.xlsx"
-        )
-        print(f"[run_multi] relating {a_name} -> {b_name} …", flush=True)
-        a = da.from_zarr(image_store, component=f"labels/{a_name}/0")
-        b = da.from_zarr(image_store, component=f"labels/{b_name}/0")
-        table = label_relations(a, b)
-
-        # label_relations() only returns a-objects that touch a b-object.
-        # Pull the full id sets so unmatched a-objects (zero overlap) and
-        # b-objects with no matches at all still get a row -- otherwise
-        # they'd silently vanish instead of counting as zero.
-        a_ids = _label_ids(a_name)
-        b_ids = _label_ids(b_name)
-
-        per_b = {b_id: {"count": 0, "overlap_voxels": 0} for b_id in b_ids}
-        for m in table.values():
-            agg = per_b.get(m["match"])
-            if agg is not None:
-                agg["count"] += 1
-                agg["overlap_voxels"] += m["overlap_voxels"]
-
-        wb = openpyxl.Workbook()
-        ws_a = wb.active
-        ws_a.title = a_name[:31]  # Excel sheet-name length limit
-        ws_a.append(
-            [
-                f"{a_name}_id",
-                f"{b_name}_id",
-                "overlap_voxels",
-                "overlap_fraction",
-            ]
-        )
-        for a_id in a_ids:
-            m = table.get(a_id)
-            if m is None:
-                ws_a.append([a_id, None, 0, 0])  # no overlap -- still counted
-            else:
-                ws_a.append(
-                    [
-                        a_id,
-                        m["match"],
-                        m["overlap_voxels"],
-                        m["overlap_fraction"],
-                    ]
-                )
-
-        ws_b = wb.create_sheet(title=b_name[:31])
-        ws_b.append([f"{b_name}_id", f"{a_name}_count", "total_overlap_voxels"])
-        for b_id in b_ids:
-            agg = per_b[b_id]
-            ws_b.append([b_id, agg["count"], agg["overlap_voxels"]])
-
-        wb.save(out_path)
-        print(
-            f"[run_multi] wrote {out_path} "
-            f"({len(a_ids)} {a_name}, {len(b_ids)} {b_name})",
-            flush=True,
-        )
+    from relate import run_relations
+
+    run_relations(work_dir, image_store, relations)
 
 
 if __name__ == "__main__":