patchworks 2.6.2__tar.gz → 2.6.3__tar.gz

{patchworks-2.6.2 → patchworks-2.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: patchworks
-Version: 2.6.2
+Version: 2.6.3
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-2.6.2 → patchworks-2.6.3}/src/patchworks/_notify.py

@@ -159,7 +159,7 @@ def slurm_mail_extra(
 
 
 def failing_step(
-    snakemake_log: Union[str, Path, None],
+    snakemake_log: "Union[str, Path, list, tuple, None]",
 ) -> "tuple[Union[str, None], Union[str, None]]":
     """Find which rule failed, and its log, from Snakemake's own log file.
 
@@ -171,9 +171,12 @@ def failing_step(
 
     Parameters
     ----------
-    snakemake_log : str or Path or None
-        Path to Snakemake's own log (the ``log`` variable inside an
-        ``onerror`` handler).
+    snakemake_log : str, Path, list, tuple, or None
+        Snakemake's own log (the ``log`` variable inside an ``onerror``
+        handler). Snakemake 8+ passes this as a list of paths (its
+        ``LoggerManager.get_logfile()`` returns ``List[str]``, even though
+        there is normally just one) rather than a single string -- take the
+        first entry.
 
     Returns
     -------
@@ -181,6 +184,8 @@ def failing_step(
         ``(rule_name, log_path)``, either of which may be None when the log
         is unreadable or records no rule error.
     """
+    if isinstance(snakemake_log, (list, tuple)):
+        snakemake_log = snakemake_log[0] if snakemake_log else None
     try:
         text = Path(snakemake_log).read_text(errors="replace")
     except (OSError, TypeError, ValueError):

{patchworks-2.6.2 → patchworks-2.6.3}/tests/test_notify.py

@@ -99,6 +99,30 @@ def test_failing_step_reads_the_rule_from_snakemakes_log(tmp_path):
     )
 
 
+def test_failing_step_accepts_snakemakes_list_log(tmp_path):
+    """Snakemake 8+ passes `log` as a list, not a bare path.
+
+    ``LoggerManager.get_logfile()`` returns ``List[str]``, and that's what
+    the `log` variable inside `onerror:` actually is. `Path(a_list)` raises
+    `TypeError`, which the old code caught and turned into `(None, None)` --
+    so this path was *always* silently falling back to the mtime guess it
+    was written to replace, on every real failure.
+    """
+    from patchworks._notify import failing_step
+
+    log = tmp_path / "sm.log"
+    log.write_text(
+        "Error in rule segment:\n"
+        "    jobid: 69\n"
+        "    output: seg/61.done\n"
+        "    log: /w/nuclei_labels/logs/segment/61.log (check log file(s))\n"
+    )
+    expected = ("segment", "/w/nuclei_labels/logs/segment/61.log")
+    assert failing_step([str(log)]) == expected
+    assert failing_step((str(log),)) == expected
+    assert failing_step([]) == (None, None)
+
+
 def test_failing_step_degrades_quietly(tmp_path):
     """It runs inside an error handler, so it must never raise itself."""
     from patchworks._notify import failing_step

{patchworks-2.6.2 → patchworks-2.6.3}/workflow/profile/slurm/config.yaml

@@ -74,7 +74,7 @@ set-resources:
     # equal to segment's mem_mb below: lower and tiles come out needlessly
     # small; higher and the sizer budgets against more host RAM than segment
     # will actually get, undoing the point of the host-RAM check.
-    mem_mb: "attempt * 32000"
+    mem_mb: "attempt * 128000"
     runtime: 120
   segment:
     # one GPU per job — this is what spreads Cellpose across GPUs.
@@ -83,15 +83,35 @@ set-resources:
     # leaves the job without a device.)
     slurm_partition: "rtx4090"
     gres: "gpu:1"
-    qos: "rtx4090-6hours" # scicore: <partition>-<duration> QOS
+    # rtx4090-6hours wasn't enough: a do_3D whole-z tile (126 planes, full
+    # footprint) can run past 6h even though most same-sized tiles finish in
+    # 2.5-3.5h -- retrying it bought nothing since `runtime` didn't scale with
+    # `attempt` the way `mem_mb` does below, so it hit the identical wall on
+    # every attempt and burned all `retries` for a job that was never going to
+    # finish in the window. rtx4090-1day is scicore's next tier up (see
+    # `sacctmgr show qos format=name,maxwall | grep rtx4090`); 24h is wide
+    # margin over anything observed so far. If a tile still times out at 24h,
+    # that's not a slow tile anymore -- it's stuck, and worth profiling on its
+    # own rather than reaching for rtx4090-1week.
+    qos: "rtx4090-1day" # scicore: <partition>-<duration> QOS
     # A job now processes `tiles_per_job` tiles sequentially, so both memory
     # and runtime scale with that setting — raise it there and re-check here.
     # The old "a tile used ~1G" note predates tile_shape: "auto", which sizes
     # tiles against the real GPU and makes them far bigger. If you raise
     # this, raise prepare's mem_mb above to match (see its comment).
-    mem_mb: "attempt * 32000"
+    #
+    # 128 GB base (256/384 GB on retry) is a wide safety margin, not a tight
+    # estimate: a do_3D tile that `auto_tile_shape_cellpose`'s 20x
+    # cellpose_memory_factor judged safe within a 24 GiB GPU budget still hit
+    # a real 32 GB host OOM, so that heuristic underestimates do_3D's actual
+    # host-RAM use by more than expected. rtx4090 nodes have ~1 TB RAM
+    # (~800 GB usable) and the QOS caps at 4 TB account-wide, so this has
+    # plenty of room -- it buys time until do_3D gets a properly measured
+    # memory factor (e.g. from `seff` on a job that completes) instead of a
+    # guess. ``ponytail:`` tighten this once real peak-RSS numbers exist.
+    mem_mb: "attempt * 128000"
     cpus_per_task: 4
-    runtime: 360 # 6 hours — must match the QOS, NOT 120 (=2h → killed early)
+    runtime: 1400 # ~23.3h — must stay under the rtx4090-1day QOS cap (24h)
   merge:
     # merge.py sizes its worker pool from the cgroup/SLURM budget rather than
     # the node's core count, and the pyramid is written zarr-natively (one