patchworks 2.3.0__tar.gz → 2.4.0__tar.gz

{patchworks-2.3.0 → patchworks-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 2.3.0
+Version: 2.4.0
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-2.3.0 → patchworks-2.4.0}/cliff.toml

@@ -32,7 +32,10 @@ commit_preprocessors = [
 ]
 commit_parsers = [
   # Merge commits are noise in a changelog — the merged commits are already listed.
+  # Anchored to the message start so a commit that merely mentions "merge dev
+  # ... main" in its own text isn't swept up too.
   { message = "^Merge ", skip = true },
+  { message = "(?i)^chore:\\s*merge dev.*main", skip = true },
   { message = "^feat", group = "✨ Features" },
   { message = "^fix", group = "🐛 Bug Fixes" },
   { message = "^perf", group = "⚡ Performance" },

{patchworks-2.3.0 → patchworks-2.4.0}/docs/examples/dog.md

@@ -147,11 +147,13 @@ example above — and the segment job deconvolves each tile with
 `pycudadecon` before running the DoG detector:
 
 ```yaml
-# config/config_cilia.yaml (excerpt)
+# config/config_cilia.yaml (excerpt) — only what differs from common.yaml,
+# which supplies the input, work_dir, tile_shape and skip_empty
 channel: 2
-tile_shape: [16, 1024, 1024]
-overlap: 30 # cover the PSF support (decon) + the DoG's high_sigma
-skip_empty: true
+# Per-axis halo [z, y, x], covering the PSF support (decon) + the DoG's
+# high_sigma. A scalar 30 would expand a [16, 1024, 1024] tile to 5.3x the
+# voxels it keeps, nearly all of it wasted z.
+overlap: [8, 30, 30]
 
 method: "custom"
 label_name: "cilia_labels"
@@ -185,11 +187,11 @@ in `dxdata`/`dxpsf` from X/Y and `dzdata`/`dzpsf` from Z. Set any of them in
     of the calibration in its config. If the store is uncalibrated the
     workflow says so and passes nothing.
 
-Run it exactly like a Cellpose config:
+Run it exactly like a Cellpose config — the shared settings come from
+`config/common.yaml`, merged in ahead of this one:
 
 ```bash
-python -m snakemake --workflow-profile profile/slurm \
-                    --configfile config/config_cilia.yaml
+python -m snakemake --workflow-profile profile/slurm --configfile config/common.yaml config/config_cilia.yaml
 ```
 
 Checklist specific to this config:
@@ -206,10 +208,13 @@ Checklist specific to this config:
   a thin intensity/threshold halo isn't enough once deconvolution is in the
   loop.
 - **`skip_empty`:** the `prepare` rule (`workflow/scripts/prepare_tiles.py`)
-  calls `estimate_empty_tiles()` before submitting any `segment` jobs,
-  regardless of `method`, so cilia/DoG runs skip background tiles exactly
-  like Cellpose runs — no extra config needed beyond `skip_empty: true`
-  (the default).
+  builds a max-pooled occupancy map and reduces it over each tile's **full**
+  footprint (`build_occupancy_map` + `tile_occupancy`) before submitting any
+  `segment` jobs, regardless of `method`. Cilia are small and often sit near
+  a tile's edge, which is precisely where the older centred-window preview
+  could miss them — this decides every tile exactly. No extra config needed
+  beyond `skip_empty: true` (the default), and the map is built once and
+  shared by every config against that store.
 - Run alongside `config_cyto.yaml`/`config_nuclei.yaml` via `config/multi.yaml`
   to also get the cilia→cell/nucleus relation — see *Relating cilia to their
   cell*, below.

{patchworks-2.3.0 → patchworks-2.4.0}/docs/guide/performance.md

@@ -74,6 +74,9 @@ extra read-back of the staged data.
 
 - `tile_shape="auto"` sizes tiles to free RAM (or VRAM with `use_gpu=True`).
 - `skip_empty=True` with `estimate_empty_tiles()` skips background tiles.
+  That preview samples a centred window, so use `build_occupancy_map()` +
+  `tile_occupancy()` when the result becomes an authoritative skip list —
+  they are exact, and it is what the Snakemake workflow uses.
 - A Dask **distributed** cluster (`make_local_cluster`) parallelises across
   workers/GPUs; patchworks then defers concurrency to the cluster.
 

{patchworks-2.3.0 → patchworks-2.4.0}/docs/guide/snakemake.md

@@ -520,7 +520,8 @@ prologue. The simplest path is a single shared env that the compute nodes see.
 | `cellpose is not installed` in a job | the job's env lacks `patchworks[cellpose]` |
 | Reading the input fails | install the matching reader (`patchworks[imaris]`/`[bioio]` + a `bioio-*`) |
 | Out of GPU memory | smaller `tile_shape`, or `do_3D: false` |
-| A job fails with an empty SLURM log | read `logs/segment/<index>.log` (per tile) or `logs/steps.log` — the real traceback is there |
+| A job fails with an empty SLURM log | read the step's own log — `logs/convert.log`, `logs/prepare.log`, `logs/segment/<batch>.log`, `logs/merge.log` — the real traceback is there |
+| A long step looks hung | every step logs progress (`… 4,200/8,064 (52%) after 31m, ~28m left`) roughly once a minute; `tail -f` the step's log above |
 | Very slow | confirm GPU is used (`nvidia-smi`); try 2-D or a lower `level` |
 
 ## How it works (for the curious)

{patchworks-2.3.0 → patchworks-2.4.0}/src/patchworks/_merge.py

@@ -23,7 +23,6 @@ import logging
 import os
 import tempfile
 from concurrent.futures import ThreadPoolExecutor
-from contextlib import nullcontext as _nullcontext
 from itertools import product as _iproduct
 from multiprocessing import Pool as _Pool
 from pathlib import Path
@@ -35,11 +34,7 @@ import zarr
 
 from ._chunks import cpu_allocation
 from ._io import zarr_compressor_kwargs
-
-try:
-    from tqdm.auto import tqdm as _tqdm
-except ImportError:
-    _tqdm = None
+from ._progress import track
 
 logger = logging.getLogger(__name__)
 
@@ -165,6 +160,7 @@ def _scan_touching_pairs(
     label_offsets: "np.ndarray | None" = None,
     n_workers: int = 1,
     has_labels: "np.ndarray | None" = None,
+    progress: bool = False,
 ) -> np.ndarray:
     """Scan chunk-boundary slabs; return (N, 2) int64 array of touching pairs.
 
@@ -259,12 +255,26 @@ def _scan_touching_pairs(
 
     nw = max(1, min(n_workers, len(tasks)))
     if nw <= 1:
-        results = [_one(t) for t in tasks]
+        results = list(
+            track(
+                (_one(t) for t in tasks),
+                "scan boundaries",
+                len(tasks),
+                enabled=progress,
+            )
+        )
     else:
         # Reads and decompression release the GIL, so threads scale here and
         # nothing has to be pickled across processes.
         with ThreadPoolExecutor(max_workers=nw) as pool:
-            results = list(pool.map(_one, tasks))
+            results = list(
+                track(
+                    pool.map(_one, tasks),
+                    "scan boundaries",
+                    len(tasks),
+                    enabled=progress,
+                )
+            )
 
     all_pairs = [r for r in results if r is not None]
     if not all_pairs:
@@ -609,6 +619,7 @@ def zarr_native_merge(
         label_offsets=offsets,
         n_workers=n_workers,
         has_labels=has_labels,
+        progress=show_progress,
     )
     logger.info(
         "zarr_native_merge: %d touching pairs → building LUT", len(pairs)
@@ -743,9 +754,9 @@ def zarr_native_merge(
             _init_worker(
                 lut_path, staged_path, staged_component, out_path, out_component
             )
-            it: Any = tasks
-            if show_progress and _tqdm is not None:
-                it = _tqdm(it, total=n_chunks, desc="relabel chunks")
+            it: Any = track(
+                tasks, "relabel chunks", n_chunks, enabled=show_progress
+            )
             for task in it:
                 _relabel_chunk_worker(task)
         else:
@@ -760,9 +771,12 @@ def zarr_native_merge(
                     out_component,
                 ),
             ) as pool:
-                it = pool.imap_unordered(_relabel_chunk_worker, tasks)
-                if show_progress and _tqdm is not None:
-                    it = _tqdm(it, total=n_chunks, desc="relabel chunks")
+                it = track(
+                    pool.imap_unordered(_relabel_chunk_worker, tasks),
+                    "relabel chunks",
+                    n_chunks,
+                    enabled=show_progress,
+                )
                 for _ in it:
                     pass
     finally:
@@ -926,9 +940,10 @@ def merge_tile_labels(
         stage_path = os.path.join(_base, "_pws_stage.zarr")
 
         import dask
-        from dask.diagnostics import ProgressBar
 
-        ctx = ProgressBar() if progress else _nullcontext()
+        from ._progress import dask_progress
+
+        ctx = dask_progress("stage tiles", progress)
         logger.info("Staging per-tile labels to %s …", stage_path)
         with ctx:
             dask.compute(

{patchworks-2.3.0 → patchworks-2.4.0}/src/patchworks/_occupancy.py

@@ -36,6 +36,7 @@ import numpy as np
 import zarr
 
 from ._chunks import cpu_allocation
+from ._progress import track
 
 logger = logging.getLogger(__name__)
 
@@ -271,13 +272,19 @@ def build_occupancy_map(
     try:
         n_workers = max(1, min(cpu_allocation(), len(regions)))
         if n_workers <= 1:
-            for starts in regions:
-                _one(starts)
+            for _ in track(
+                (_one(starts) for starts in regions),
+                "occupancy map",
+                len(regions),
+            ):
+                pass
         else:
             # Reads and decompression release the GIL, so threads are enough
             # and there is no worker payload to pickle.
             with ThreadPoolExecutor(max_workers=n_workers) as pool:
-                for _ in pool.map(_one, regions):
+                for _ in track(
+                    pool.map(_one, regions), "occupancy map", len(regions)
+                ):
                     pass
         dst.attrs["block"] = list(block)
         dst.attrs["level"] = int(level)

patchworks-2.4.0/src/patchworks/_progress.py

@@ -0,0 +1,184 @@
+"""Progress reporting that survives being written to a log file.
+
+Every long step in the workflow runs unattended in a batch job, where the
+output is read hours later out of a file. A redrawing bar (dask's
+``ProgressBar``, ``tqdm``) collapses into one enormous unreadable line there,
+so the default here is periodic log records instead -- and a bar only when
+someone is actually watching a terminal.
+
+The interval matters more than the precision: the job runs for hours, and the
+question being answered is "is this working or hung?", not "exactly how far".
+"""
+
+from __future__ import annotations
+
+import logging
+import sys
+import time
+from contextlib import nullcontext
+from typing import Iterable, Iterator, TypeVar
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+
+# Often enough to tell progress from a hang, rare enough that a six-hour job
+# leaves a log you can still read.
+PROGRESS_INTERVAL_S = 60.0
+
+
+def is_interactive() -> bool:
+    """True when someone is plausibly watching a terminal."""
+    return bool(getattr(sys.stderr, "isatty", lambda: False)())
+
+
+def format_eta(done: int, total: int, elapsed: float) -> str:
+    """Rough remaining time from a linear extrapolation.
+
+    Parameters
+    ----------
+    done, total : int
+        Units finished and expected.
+    elapsed : float
+        Seconds spent so far.
+
+    Returns
+    -------
+    str
+        A short human-readable duration, or ``"?"`` when it cannot be
+        estimated yet.
+    """
+    if done <= 0 or done >= total:
+        return "?"
+    left = elapsed / done * (total - done)
+    if left < 90:
+        return f"{left:.0f}s"
+    if left < 5400:
+        return f"{left / 60:.0f}m"
+    return f"{left / 3600:.1f}h"
+
+
+def log_progress(label: str, done: int, total: int, started: float) -> None:
+    """Emit one progress line.
+
+    Parameters
+    ----------
+    label : str
+        What is being worked on, e.g. ``"image.zarr/0"``.
+    done, total : int
+        Units finished and expected.
+    started : float
+        ``time.monotonic()`` when the work began.
+    """
+    elapsed = time.monotonic() - started
+    logger.info(
+        "%s: %s/%s (%.0f%%) after %.0fm, ~%s left",
+        label,
+        f"{done:,}",
+        f"{total:,}",
+        100.0 * done / max(1, total),
+        elapsed / 60,
+        format_eta(done, total, elapsed),
+    )
+
+
+def track(
+    iterable: Iterable[T],
+    label: str,
+    total: int,
+    *,
+    enabled: bool = True,
+) -> Iterator[T]:
+    """Yield from *iterable*, reporting progress as it goes.
+
+    Uses ``tqdm`` when attached to a terminal and periodic log lines
+    otherwise, so the same call is right in a notebook and in a SLURM job.
+
+    Parameters
+    ----------
+    iterable : iterable
+        The work to iterate. Consumed lazily, so this is safe over
+        ``imap_unordered``.
+    label : str
+        Description of the work.
+    total : int
+        Expected number of items, used for the percentage and the ETA.
+    enabled : bool, optional
+        Set ``False`` to pass items straight through. Default ``True``.
+
+    Yields
+    ------
+    object
+        The items of *iterable*, unchanged.
+    """
+    if not enabled:
+        yield from iterable
+        return
+
+    if is_interactive():
+        try:
+            from tqdm.auto import tqdm
+
+            yield from tqdm(iterable, total=total, desc=label)
+            return
+        except ImportError:
+            pass
+
+    started = time.monotonic()
+    last = started
+    done = 0
+    logger.info("%s: starting (%s items)", label, f"{total:,}")
+    for item in iterable:
+        yield item
+        done += 1
+        now = time.monotonic()
+        if now - last >= PROGRESS_INTERVAL_S:
+            last = now
+            log_progress(label, done, total, started)
+    log_progress(label, done, total, started)
+
+
+def dask_progress(label: str, enabled: bool = True):
+    """Progress context manager for a dask computation.
+
+    Returns
+    -------
+    contextmanager
+        ``ProgressBar`` on a terminal, a periodically-logging callback
+        otherwise, or a no-op when *enabled* is false.
+    """
+    if not enabled:
+        return nullcontext()
+
+    logger.info("writing %s …", label)
+    if is_interactive():
+        from dask.diagnostics import ProgressBar
+
+        return ProgressBar()
+
+    from dask.callbacks import Callback
+
+    class _LogProgress(Callback):
+        """Count finished dask tasks and log every PROGRESS_INTERVAL_S."""
+
+        def _start_state(self, dsk, state):
+            self._total = sum(
+                len(state[k])
+                for k in ("ready", "waiting", "running", "finished")
+            )
+            self._done = 0
+            self._t0 = time.monotonic()
+            self._last = self._t0
+
+        def _posttask(self, key, result, dsk, state, worker_id):
+            self._done += 1
+            now = time.monotonic()
+            if now - self._last >= PROGRESS_INTERVAL_S:
+                self._last = now
+                log_progress(label, self._done, self._total, self._t0)
+
+        def _finish(self, dsk, state, errored):
+            if not errored:
+                log_progress(label, self._total, self._total, self._t0)
+
+    return _LogProgress()

{patchworks-2.3.0 → patchworks-2.4.0}/src/patchworks/plugins/ome_zarr.py

@@ -48,6 +48,7 @@ from __future__ import annotations
 import glob
 import logging
 import math
+import time as _time
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import nullcontext as _nullcontext
 from itertools import product as _iproduct
@@ -59,6 +60,10 @@ import numpy as np
 import zarr
 
 from .._chunks import cpu_allocation
+from .._progress import (
+    PROGRESS_INTERVAL_S as _PROGRESS_INTERVAL_S,
+)
+from .._progress import dask_progress, log_progress
 from .._io import load_ome_zarr, zarr_compressor_kwargs
 
 logger = logging.getLogger(__name__)
@@ -423,6 +428,8 @@ def _stream_strided_level(
     dst: "zarr.Array",
     strides: tuple[int, ...],
     n_workers: int = 4,
+    label: str = "level",
+    progress: bool = True,
 ) -> None:
     """Write *dst* as the strided subsample of *src*, one chunk at a time.
 
@@ -457,13 +464,27 @@ def _stream_strided_level(
         dst[out_sl] = np.asarray(src[src_sl])[take]
 
     indices = list(_iproduct(*[range(g) for g in grid]))
+    total = len(indices)
+    started = _time.monotonic()
+    last = started
+
+    def _tick(done: int) -> None:
+        nonlocal last
+        now = _time.monotonic()
+        if progress and now - last >= _PROGRESS_INTERVAL_S:
+            last = now
+            log_progress(label, done, total, started)
+
     if n_workers <= 1:
-        for idx in indices:
+        for done, idx in enumerate(indices, 1):
             _one(idx)
-        return
-    with ThreadPoolExecutor(max_workers=n_workers) as pool:
-        for _ in pool.map(_one, indices):
-            pass
+            _tick(done)
+    else:
+        with ThreadPoolExecutor(max_workers=n_workers) as pool:
+            for done, _ in enumerate(pool.map(_one, indices), 1):
+                _tick(done)
+    if progress:
+        log_progress(label, total, total, started)
 
 
 # One chunk is one file without sharding. A shared cluster filesystem starts
@@ -531,29 +552,8 @@ def _bounded_scheduler(arr: da.Array):
 
 
 def _progress_ctx(progress: bool, label: str):
-    """Return a progress-bar context manager.
-
-    Parameters
-    ----------
-    progress : bool
-        Whether to show a dask progress bar.
-    label : str
-        Name logged just before the bar.
-
-    Returns
-    -------
-    contextmanager
-        A ``ProgressBar`` when *progress* is set, else a no-op
-        context manager.
-    """
-    if not progress:
-        from contextlib import nullcontext
-
-        return nullcontext()
-    from dask.diagnostics import ProgressBar
-
-    logger.info("writing %s …", label)
-    return ProgressBar()
+    """Progress context for a long dask write; see patchworks._progress."""
+    return dask_progress(label, progress)
 
 
 def _to_zarr_level(
@@ -786,7 +786,12 @@ def _write_pyramid(
                     ),
                 )
             _stream_strided_level(
-                src_arr, dst_arr, strides, n_workers=cpu_allocation()
+                src_arr,
+                dst_arr,
+                strides,
+                n_workers=cpu_allocation(),
+                label=f"{Path(group_path).name}/{i}",
+                progress=progress,
             )
         scale = [base_scale[k] * (strides[k] ** i) for k in range(len(axes))]
         datasets.append(_dataset(str(i), scale))

patchworks-2.4.0/tests/test_progress.py

@@ -0,0 +1,60 @@
+"""Tests for log-friendly progress reporting."""
+
+import logging
+
+from patchworks import _progress
+
+
+def test_track_yields_everything_unchanged():
+    """Progress reporting must never alter or drop the work it wraps."""
+    items = list(range(50))
+    assert list(_progress.track(iter(items), "x", len(items))) == items
+    # ...including when disabled, which is the passthrough path.
+    assert (
+        list(_progress.track(iter(items), "x", len(items), enabled=False))
+        == items
+    )
+
+
+def test_track_logs_on_a_non_tty(monkeypatch, caplog):
+    """A batch job gets log records, not a redrawing bar.
+
+    The whole point: a carriage-returning bar collapses a SLURM log into one
+    unreadable line, which is why convert ran silent rather than use one.
+    """
+    monkeypatch.setattr(_progress, "is_interactive", lambda: False)
+    # Force every item to report, instead of waiting out the real interval.
+    monkeypatch.setattr(_progress, "PROGRESS_INTERVAL_S", -1.0)
+    with caplog.at_level(logging.INFO, logger=_progress.logger.name):
+        list(_progress.track(iter(range(3)), "merge chunks", 3))
+
+    messages = [r.getMessage() for r in caplog.records]
+    assert any("starting" in m for m in messages)
+    assert any("merge chunks" in m and "%" in m for m in messages)
+    # No carriage returns: that is what makes it readable in a file.
+    assert not any("\r" in m for m in messages)
+
+
+def test_track_reports_the_final_count_even_when_quiet():
+    """A run shorter than one interval must still say it finished."""
+    records = []
+    handler = logging.Handler()
+    handler.emit = records.append
+    _progress.logger.addHandler(handler)
+    _progress.logger.setLevel(logging.INFO)
+    try:
+        list(_progress.track(iter(range(2)), "quick", 2))
+    finally:
+        _progress.logger.removeHandler(handler)
+    assert any("2/2" in r.getMessage() for r in records)
+
+
+def test_eta_is_sane_and_degrades_gracefully():
+    """An ETA is a linear extrapolation, and refuses to invent one."""
+    assert _progress.format_eta(0, 100, 10.0) == "?"  # nothing measured yet
+    assert _progress.format_eta(100, 100, 10.0) == "?"  # already done
+    # Half done in 60 s → roughly another 60 s.
+    assert _progress.format_eta(50, 100, 60.0) == "60s"
+    # Units scale so a six-hour job does not report "21600s".
+    assert _progress.format_eta(1, 100, 60.0).endswith(("m", "h"))
+    assert _progress.format_eta(1, 1000, 600.0).endswith("h")

{patchworks-2.3.0 → patchworks-2.4.0}/tests/test_run_multi.py

@@ -121,3 +121,40 @@ def test_shipped_multi_configs_are_consistent():
     for path in paths:
         own = yaml.safe_load(path.read_text())
         assert not set(own) & set(_CONVERT_KEYS), path.name
+
+
+def _workflow_dir() -> Path:
+    return Path(__file__).resolve().parents[1] / "workflow"
+
+
+def test_occupancy_is_a_submitted_rule_not_a_localrule():
+    """The occupancy build must never run on the submit host.
+
+    It streams the entire image. Doing that in the run_multi driver ran it on
+    a login node, where a multi-terabyte read is killed with no traceback --
+    the run just returned to the prompt. Only fetch_model may be local (it
+    needs network); everything else has to get a real allocation.
+    """
+    wf = _workflow_dir()
+    snakefile = (wf / "Snakefile").read_text()
+    local_block = snakefile.split("localrules:")[1].split("rule ")[0]
+    local = {
+        line.strip().rstrip(",")
+        for line in local_block.splitlines()
+        if line.strip() and not line.strip().startswith("#")
+    }
+    assert local == {"fetch_model"}, local
+
+    rules = (wf / "rules" / "convert.smk").read_text()
+    assert "rule occupancy:" in rules
+
+
+def test_occupancy_is_not_rebuilt_by_the_driver():
+    """run_multi must ask Snakemake for the map, not build it in-process.
+
+    An in-process build bypasses the scheduler entirely, which is how it ended
+    up on the login node.
+    """
+    src = (_workflow_dir() / "scripts" / "run_multi.py").read_text()
+    assert "build_occupancy_map(" not in src
+    assert "occupancy.zarr" in src

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/profile/slurm/config.yaml

@@ -56,10 +56,17 @@ set-resources:
     mem_mb: "attempt * 64000"
     cpus_per_task: 32
     runtime: 360
+  occupancy:
+    # One streaming pass over the whole image, max-pooling each brick. Reads a
+    # lot but holds little: memory is bounded by the worker count times one
+    # region, not by the image. Runtime tracks the read, so give it room --
+    # this is a multi-terabyte scan on a real dataset.
+    mem_mb: "attempt * 32000"
+    cpus_per_task: 16
+    runtime: 360
   prepare:
-    # Also builds the max-pooled occupancy map on first use for an image.zarr
-    # that predates it (one bounded streaming pass, then shared by every
-    # config against that store).
+    # Cheap now: it reduces the already-built occupancy map over each tile's
+    # footprint instead of touching the image.
     mem_mb: "attempt * 32000"
     runtime: 120
   segment:

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/rules/common.smk

@@ -8,6 +8,13 @@ IMAGE = f"{WORK}/image.zarr"
 # avoids wiping the whole store on a re-run (same trick as imcf/sopa).
 IMAGE_OK = f"{IMAGE}/zarr.json"
 
+# Max-pooled occupancy summary, a sibling of the image (not a node inside it,
+# which zarr would refuse to walk). Shared by every config against this image,
+# so it is keyed on the image and the level rather than on label_name.
+OCCUPANCY = f"{WORK}/image.occupancy.zarr/{int(config.get('level', 0))}"
+OCCUPANCY_OK = f"{OCCUPANCY}/zarr.json"
+OCCUPANCYLOG = f"{WORK}/logs/occupancy.log"
+
 # Everything below is per-segmentation, namespaced under WORK/<label_name>/, so
 # running the workflow twice with two configs (different label_name, e.g.
 # "nuclei_labels" and "cell_labels") against the *same* work_dir never
@@ -24,10 +31,14 @@ STAGE = f"{RUN}/stage.zarr"
 STAGE_OK = f"{STAGE}.done"
 
 
-# Logs: one shared file for the sequential CPU steps (convert/prepare/merge),
-# one file per tile for the GPU segment jobs.
+# Logs: one file per step. They used to share a single steps.log, but
+# Snakemake clears a rule's declared log before the job runs, so each step
+# wiped the previous one's output -- by the time a run finished, only the last
+# step's log survived and a failure earlier on left nothing to read.
 LOGS = f"{RUN}/logs"
-STEPLOG = f"{LOGS}/steps.log"
+CONVERTLOG = f"{LOGS}/convert.log"
+PREPARELOG = f"{LOGS}/prepare.log"
+MERGELOG = f"{LOGS}/merge.log"
 
 # Marker that the segmentation model is cached locally. Produced by a local
 # rule (runs on the networked submit host) so offline GPU nodes never download.

patchworks-2.4.0/workflow/rules/convert.smk

@@ -0,0 +1,24 @@
+# Convert the input to a pyramidal OME-ZARR.
+
+rule convert:
+    output:
+        # marker file inside the store; existence => skip re-conversion.
+        IMAGE_OK,
+    log:
+        CONVERTLOG,
+    script:
+        "../scripts/convert.py"
+
+
+# Build the occupancy map as a real job. It streams the whole image, so doing
+# it in the run_multi driver ran it on the login node, where a multi-terabyte
+# read is killed with no traceback. Built once and reused by every config.
+rule occupancy:
+    input:
+        IMAGE_OK,
+    output:
+        OCCUPANCY_OK,
+    log:
+        OCCUPANCYLOG,
+    script:
+        "../scripts/build_occupancy.py"

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/rules/merge.smk

@@ -7,6 +7,6 @@ rule merge:
     output:
         touch(f"{RUN}/labels.done"),
     log:
-        STEPLOG,
+        MERGELOG,
     script:
         "../scripts/merge.py"

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/rules/segment.smk

@@ -18,11 +18,15 @@ rule fetch_model:
 checkpoint prepare:
     input:
         IMAGE_OK,
+        # Depend on the map rather than building it inline: it streams the
+        # whole image, and several configs' prepare steps run concurrently, so
+        # inline each would stream the volume and all but one discard it.
+        OCCUPANCY_OK,
     output:
         tiles=TILES,
         stage=touch(STAGE_OK),
     log:
-        STEPLOG,
+        PREPARELOG,
     script:
         "../scripts/prepare_tiles.py"
 

patchworks-2.4.0/workflow/scripts/build_occupancy.py

@@ -0,0 +1,33 @@
+"""Snakemake script: build the max-pooled occupancy map for one level.
+
+Its own rule, rather than work done inside ``run_multi``, because it streams
+the entire image: doing that in the driver process runs it on the **login
+node**, where a multi-terabyte read is killed without a traceback. As a rule
+it is submitted like any other job and gets a real allocation.
+
+Built once per image and shared by every config that segments it -- otherwise
+the concurrently-running ``prepare`` steps would each stream the whole volume
+and all but one would throw the result away.
+"""
+
+from patchworks import block_for_tile, build_occupancy_map
+
+from _pw import start_log
+
+start_log(snakemake.log[0])  # noqa: F821
+cfg = snakemake.config  # noqa: F821
+
+image_store = str(snakemake.input[0]).removesuffix("/zarr.json")  # noqa: F821
+level = int(cfg.get("level", 0))
+
+# Sizing the block from the tile keeps the map discriminating: a block as
+# coarse as the tile itself would make every tile test occupied.
+tile_shape = cfg.get("tile_shape")
+kwargs = (
+    {"block": block_for_tile(tuple(tile_shape))}
+    if isinstance(tile_shape, (list, tuple))
+    else {}
+)
+
+path = build_occupancy_map(image_store, level=level, **kwargs)
+print(f"[patchworks] occupancy map ready at {path}", flush=True)

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/scripts/convert.py

@@ -30,6 +30,9 @@ to_ome_zarr(
     chunks=tuple(chunks) if chunks else None,
     shard=bool(cfg.get("shard", False)),
     reuse_pyramid=bool(cfg.get("reuse_pyramid", False)),
-    progress=False,
+    # Progress goes to the log as periodic lines, not a redrawing bar:
+    # this runs for hours in a batch job where silence is indistinguishable
+    # from a hang.
+    progress=True,
     overwrite=True,
 )

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/scripts/merge.py

@@ -91,7 +91,8 @@ _, n_objects = merge_tile_labels(
     output_chunks=out_chunks,
     sequential_labels=cfg.get("sequential_labels", True),
     n_workers=cfg.get("merge_workers") or default_workers,
-    progress=False,
+    # Periodic log lines rather than a bar (see convert.py).
+    progress=True,
     return_count=True,
     label_counts=label_counts,
 )
@@ -100,7 +101,7 @@ group = register_labels(
     label_name,
     n_levels=int(cfg.get("pyramid_levels", 5)),
     downscale=int(cfg.get("pyramid_downscale", 2)),
-    progress=False,
+    progress=True,
     n_objects=n_objects,
 )
 

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/scripts/run_multi.py

@@ -272,6 +272,10 @@ def main() -> None:
     seg_cfgs = [{**common_cfg, **_load_yaml(p)} for p in seg_config_paths]
     work_dir = _validate_configs(seg_config_paths, seg_cfgs)
     image_store = f"{work_dir}/image.zarr"
+    # Shared by every config, hence keyed on the image and level, not on a
+    # label_name. Levels are validated identical across configs below.
+    _level = int(seg_cfgs[0].get("level", 0))
+    occupancy_store = f"{work_dir}/image.occupancy.zarr/{_level}"
 
     # Each phase gets its own Snakemake state directory (the lock lives in the
     # working directory, not the config), so unlocking has to cover all of
@@ -311,7 +315,15 @@ def main() -> None:
             cores=args.cores,
             dry_run=args.dry_run,
             state_dir=Path(work_dir) / ".snakemake_convert",
-            targets=[f"{image_store}/zarr.json"],
+            # Both in one phase-A call so they run as SLURM jobs. The
+            # occupancy map streams the whole image; building it here in the
+            # driver ran it on the login node, where the read is killed
+            # without a traceback. It is shared by every config, so it must
+            # not be left to the concurrent `prepare` steps either.
+            targets=[
+                f"{image_store}/zarr.json",
+                f"{occupancy_store}/zarr.json",
+            ],
             jobname_prefix=slurm_jobname_prefix("convert"),
             common=common_path,
         ),
@@ -327,31 +339,6 @@ def main() -> None:
         )
         sys.exit(rc)
 
-    # Still phase A: build the occupancy map here too. Every config's `prepare`
-    # needs it, and they are about to run concurrently -- so leaving it to them
-    # means all of them stream the whole image, and all but one throw the
-    # result away. It covers every channel, so one build serves them all.
-    if not args.dry_run:
-        from patchworks import block_for_tile, build_occupancy_map
-
-        # tile_shape is validated identical across configs, so one block suits
-        # them all. Sizing it from the tile keeps the map discriminating: a
-        # block as coarse as the tile makes every tile test occupied.
-        tile_shape = seg_cfgs[0].get("tile_shape")
-        block = (
-            block_for_tile(tile_shape)
-            if isinstance(tile_shape, (list, tuple))
-            else None
-        )
-        levels = {int(cfg.get("level", 0)) for cfg in seg_cfgs}
-        for level in sorted(levels):
-            print(
-                f"[run_multi] building occupancy map for level {level} …",
-                flush=True,
-            )
-            kwargs = {"block": block} if block else {}
-            build_occupancy_map(image_store, level=level, **kwargs)
-
     # Phase B: the segmentations touch disjoint files under
     # work_dir/<label_name>/, so run them together and let the GPU partition
     # stay busy instead of idling through each config's prepare and merge.

{patchworks-2.3.0 → patchworks-2.4.0}/workflow/scripts/segment_tile.py

@@ -8,6 +8,7 @@ batch writes disjoint chunks of the stage store, so batches never collide.
 """
 
 import json
+import time
 
 from patchworks import stage_tile
 
@@ -33,7 +34,9 @@ component = manifest.get("target_component", "staged")
 tile_shape = tuple(manifest["tile_shape"])
 
 counts = {}
-for index in indices:
+batch_started = time.monotonic()
+for n, index in enumerate(indices, 1):
+    started = time.monotonic()
     counts[index] = stage_tile(
         image,
         fn,
@@ -44,7 +47,15 @@ for index in indices:
         overlap=manifest["overlap"],
         component=component,
     )
-    print(f"[patchworks] segmented tile {index}: {counts[index]} label(s)")
+    # The per-tile time is what `tiles_per_job` has to be sized from: a job's
+    # wall time is roughly N x this, and it must stay inside the QOS ceiling.
+    # The first tile in a batch also carries the model load, so it runs long.
+    took = time.monotonic() - started
+    print(
+        f"[patchworks] tile {index} ({n}/{len(indices)}): "
+        f"{counts[index]} label(s) in {took:.0f}s",
+        flush=True,
+    )
 
 # The marker carries each tile's label count. That is what lets merge derive
 # every tile's global id range with a cumulative sum instead of streaming the
@@ -52,4 +63,8 @@ for index in indices:
 # down instead of throwing them away.
 with open(snakemake.output[0], "w") as fh:  # noqa: F821
     json.dump({"batch": batch, "counts": counts}, fh)
-print(f"[patchworks] batch {batch}: {len(indices)} tile(s) done")
+print(
+    f"[patchworks] batch {batch}: {len(indices)} tile(s) done in "
+    f"{(time.monotonic() - batch_started) / 60:.1f}m",
+    flush=True,
+)

patchworks-2.3.0/workflow/rules/convert.smk

@@ -1,10 +0,0 @@
-# Convert the input to a pyramidal OME-ZARR.
-
-rule convert:
-    output:
-        # marker file inside the store; existence => skip re-conversion.
-        IMAGE_OK,
-    log:
-        STEPLOG,
-    script:
-        "../scripts/convert.py"