patchworks 2.5.0__tar.gz → 2.5.2__tar.gz

{patchworks-2.5.0 → patchworks-2.5.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 2.5.0
+Version: 2.5.2
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-2.5.0 → patchworks-2.5.2}/docs/guide/snakemake.md

@@ -475,6 +475,24 @@ These cover what SLURM cannot: a local run with no scheduler at all, and
 failures where the useful content is the Python traceback rather than an exit
 code.
 
+Check it works without waiting for a multi-hour step:
+
+```bash
+python scripts/run_multi.py --config config/multi.yaml --test-email
+```
+
+It prints the address it actually resolved from the merged config, the exact
+`sbatch` flags the jobs will carry, and whether a test message was accepted —
+which separates "the address never reached the config" from "it did and the
+mail was dropped downstream". Those look identical otherwise: no email either
+way.
+
+!!! warning "No mail from `segment`"
+    `segment` is excluded on purpose, so a run that is only segmenting tiles
+    sends nothing. Mail comes from `convert`, `occupancy` and `merge`, plus
+    the workflow-level success/failure message — if those already completed
+    before you set the address, there is nothing left in the run to mail you.
+
 !!! note "Delivery is best-effort, by design"
     A notification can never fail a run. If no local `sendmail` exists and no
     SMTP server answers on localhost, the failure is logged as a warning and

{patchworks-2.5.0 → patchworks-2.5.2}/src/patchworks/_distributed.py

@@ -213,6 +213,22 @@ def stage_tile(
         trims.append((s.start - lo, hi - s.stop))
     block = np.asarray(image[tuple(expanded)])
     out = np.asarray(fn(block))
+    if out.shape != block.shape:
+        # Caught here rather than 6 frames deep in zarr's codec pipeline as
+        # "could not broadcast input array from shape (13,1020,1020) into
+        # shape (14,1024,1024)", which says nothing about which function is
+        # at fault. A segmentation function must label the voxels it was
+        # given: the halo trim and the destination slice are both computed
+        # from the tile's geometry, so a different shape has no defined
+        # placement.
+        name = getattr(fn, "__name__", type(fn).__name__)
+        raise ValueError(
+            f"segmentation function {name!r} returned shape {out.shape} for "
+            f"a tile of shape {block.shape} (tile {index}). It must return "
+            "one label per input voxel. Some deconvolution backends crop "
+            "their output -- pad or centre it back to the input shape before "
+            "returning."
+        )
     sel = tuple(
         slice(left, out.shape[i] - right)
         for i, (left, right) in enumerate(trims)

{patchworks-2.5.0 → patchworks-2.5.2}/src/patchworks/_io.py

@@ -95,7 +95,61 @@ def load_ome_zarr(
 
     arr = da.from_zarr(str(store_path), component=path, chunks=zarr_chunks)
     if channel is not None:
-        arr = arr[channel]
+        arr = _select_channel(arr, channel, _ms[0], store_path)
+    return arr
+
+
+def _select_channel(arr, channel: int, multiscale: dict, store_path):
+    """Index *arr*'s channel axis, or leave it alone when there isn't one.
+
+    ``arr[channel]`` used to be applied unconditionally, so on a
+    single-channel store written as plain ``zyx`` the default ``channel: 0``
+    silently sliced away **z** instead. The result stayed a valid array, just
+    one dimension short, and surfaced much later as a tile-count mismatch
+    against the occupancy grid rather than as anything about channels.
+
+    Parameters
+    ----------
+    arr : da.Array
+        The full array as stored.
+    channel : int
+        Requested channel index.
+    multiscale : dict
+        The store's multiscales entry, read for its ``axes``.
+    store_path : str or Path
+        Only used in messages.
+
+    Returns
+    -------
+    da.Array
+        *arr* with the channel axis indexed away, or unchanged when the store
+        has no channel axis and channel 0 was requested.
+    """
+    axes = [
+        (a.get("name") if isinstance(a, dict) else a) or ""
+        for a in (multiscale.get("axes") or [])
+    ]
+    if "c" in axes:
+        idx = axes.index("c")
+        return arr[(slice(None),) * idx + (channel,)]
+
+    # No axes metadata: fall back to shape. A 4-D array is c,z,y,x by the
+    # convention this package writes; a 3-D one is z,y,x.
+    if not axes and arr.ndim >= 4:
+        return arr[channel]
+
+    if channel:
+        raise ValueError(
+            f"channel={channel} was requested but {store_path!r} has no "
+            f"channel axis (axes={axes or 'unknown'}, shape={arr.shape}). "
+            "Set channel: null in the config for a single-channel image."
+        )
+    logger.info(
+        "%s has no channel axis; ignoring channel=0 and using the whole "
+        "array (shape %s).",
+        store_path,
+        arr.shape,
+    )
     return arr
 
 

{patchworks-2.5.0 → patchworks-2.5.2}/src/patchworks/_notify.py

@@ -14,6 +14,7 @@ turn a real error into a confusing one about email.
 from __future__ import annotations
 
 import logging
+import re
 import shutil
 import socket
 import subprocess
@@ -155,3 +156,44 @@ def slurm_mail_extra(
     # Keep SLURM's own order, not the config's, so the string is stable.
     types = [mapping[k] for k in ("start", "finish", "error") if k in chosen]
     return f"--mail-type={','.join(types)} --mail-user={email}"
+
+
+def failing_step(
+    snakemake_log: Union[str, Path, None],
+) -> "tuple[Union[str, None], Union[str, None]]":
+    """Find which rule failed, and its log, from Snakemake's own log file.
+
+    Guessing from step-log timestamps does not work: a `segment` failure
+    leaves `prepare.log` as the most recently written of the sequential
+    steps, so a mail built that way quotes a log that *succeeded* and names
+    the wrong step. Snakemake records the failing rule and the exact log path
+    it used, so read that instead.
+
+    Parameters
+    ----------
+    snakemake_log : str or Path or None
+        Path to Snakemake's own log (the ``log`` variable inside an
+        ``onerror`` handler).
+
+    Returns
+    -------
+    tuple
+        ``(rule_name, log_path)``, either of which may be None when the log
+        is unreadable or records no rule error.
+    """
+    try:
+        text = Path(snakemake_log).read_text(errors="replace")
+    except (OSError, TypeError, ValueError):
+        return None, None
+
+    blocks = text.split("Error in rule ")
+    if len(blocks) < 2:
+        return None, None
+    last = blocks[-1]
+    rule = re.match(r"(\S+?):", last)
+    # The log: line inside that error block points at the step's own log.
+    path = re.search(r"^\s*log:\s*(\S+?)(?:,|\s|$)", last, re.M)
+    return (
+        rule.group(1) if rule else None,
+        path.group(1) if path else None,
+    )

{patchworks-2.5.0 → patchworks-2.5.2}/src/patchworks/plugins/dog.py

@@ -238,6 +238,61 @@ def _run(block: np.ndarray, dog_dict: dict[str, Any]) -> np.ndarray:
     )
 
 
+def _restore_shape(arr: np.ndarray, shape: tuple[int, ...]) -> np.ndarray:
+    """Centre *arr* back into an array of *shape*, cropping or edge-padding.
+
+    Deconvolution must not change the field of view: patchworks writes the
+    result into a destination slice derived from the tile's geometry, so one
+    label per input voxel is required.
+
+    Centring is the right correction for a symmetric crop, which is what
+    apodisation produces. The discrepancies observed are small (a voxel in z,
+    a few in x/y) and land inside the halo, which is discarded anyway -- so
+    the labels that survive the trim are unaffected. It is logged at WARNING
+    with the exact shapes so a larger, non-symmetric crop cannot pass
+    silently.
+
+    Parameters
+    ----------
+    arr : np.ndarray
+        The deconvolved volume.
+    shape : tuple of int
+        The shape it must be returned at (the input tile's).
+
+    Returns
+    -------
+    np.ndarray
+        An array of exactly *shape*.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> _restore_shape(np.ones((13, 1020)), (14, 1024)).shape
+    (14, 1024)
+    """
+    logger.warning(
+        "deconvolution returned %s for a %s input; re-centring to the input "
+        "shape. patchworks needs one label per input voxel. A large or "
+        "asymmetric difference here would shift labels -- check the PSF and "
+        "voxel sizes if this is more than a few voxels.",
+        arr.shape,
+        shape,
+    )
+    # Crop first, so an axis that grew is handled before padding the rest.
+    crop = tuple(
+        slice((a - s) // 2, (a - s) // 2 + s) if a > s else slice(None)
+        for a, s in zip(arr.shape, shape)
+    )
+    arr = arr[crop]
+    pad = tuple(
+        ((s - a) // 2, s - a - (s - a) // 2) if a < s else (0, 0)
+        for a, s in zip(arr.shape, shape)
+    )
+    if any(lo or hi for lo, hi in pad):
+        arr = np.pad(arr, pad, mode="edge")
+    return arr
+
+
 def _segment_once(
     block: np.ndarray, dog_dict: dict[str, Any], use_gpu: bool
 ) -> np.ndarray:
@@ -253,7 +308,16 @@ def _segment_once(
         # in the per-tile timing that tile_process logs.
         from pycudadecon import decon
 
+        before = img.shape
         img = decon(images=img, **decon_kwargs)
+        if img.shape != before:
+            # cudaDecon returns a slightly smaller volume for some input
+            # sizes (e.g. (14,1024,1024) -> (13,1020,1020) on an edge tile).
+            # patchworks needs one label per input voxel: the halo trim and
+            # the destination slice are both derived from the tile geometry,
+            # so a shrunken result has nowhere to go and used to surface as
+            # an unreadable broadcast error from inside zarr.
+            img = _restore_shape(img, before)
 
     if use_gpu:
         import cupy as cp

{patchworks-2.5.0 → patchworks-2.5.2}/tests/test_core.py

@@ -357,3 +357,35 @@ def test_tile_process_max_workers():
     arr = da.from_array(_make_image((2, 32, 32)), chunks=(1, 32, 32))
     result = tile_process(arr, _label_fn, max_workers=1).compute()
     assert result.shape == (2, 32, 32)
+
+
+def test_channel_selection_respects_the_stores_axes(tmp_path):
+    """`channel: 0` on a store with no channel axis must not slice away z.
+
+    arr[channel] was applied unconditionally, so a single-channel image
+    written as plain zyx silently lost its z axis. The array stayed valid,
+    just one dimension short, and only surfaced later as a tile count that
+    disagreed with the occupancy grid.
+    """
+    import dask.array as da
+    import pytest
+
+    from patchworks import load_ome_zarr
+    from patchworks.plugins.ome_zarr import to_ome_zarr
+
+    vol = da.zeros((8, 64, 64), chunks=(4, 32, 32), dtype="uint16")
+    store = str(tmp_path / "zyx.zarr")
+    to_ome_zarr(vol, store, axes="zyx", n_levels=1, progress=False)
+
+    # channel 0 on a zyx store: keep the whole volume, do not index axis 0.
+    assert load_ome_zarr(store, channel=0, level=0).shape == (8, 64, 64)
+    assert load_ome_zarr(store, channel=None, level=0).shape == (8, 64, 64)
+    # A non-zero channel really is a mistake here, and is named as one.
+    with pytest.raises(ValueError, match="no channel axis"):
+        load_ome_zarr(store, channel=2, level=0)
+
+    # A czyx store still selects the channel as before.
+    vol4 = da.zeros((3, 8, 64, 64), chunks=(1, 4, 32, 32), dtype="uint16")
+    store4 = str(tmp_path / "czyx.zarr")
+    to_ome_zarr(vol4, store4, axes="czyx", n_levels=1, progress=False)
+    assert load_ome_zarr(store4, channel=1, level=0).shape == (8, 64, 64)

{patchworks-2.5.0 → patchworks-2.5.2}/tests/test_dog.py

@@ -107,3 +107,56 @@ def test_explicit_decon_kwargs_win_over_the_calibration(monkeypatch):
     assert captured["dxdata"] == 0.1  # filled from the calibration
     assert captured["dzdata"] == 0.2
     assert captured["dzpsf"] == 0.2
+
+
+def test_restore_shape_recentres_a_cropped_decon():
+    """cudaDecon can hand back a smaller volume than it was given.
+
+    Observed on a real edge tile: (14, 1024, 1024) in, (13, 1020, 1020) out.
+    patchworks needs one label per input voxel, so the field of view has to be
+    restored before the DoG step.
+    """
+    from patchworks.plugins.dog import _restore_shape
+
+    arr = np.arange(13 * 1020 * 1020, dtype="float32").reshape(13, 1020, 1020)
+    out = _restore_shape(arr, (14, 1024, 1024))
+    assert out.shape == (14, 1024, 1024)
+    # The original content is preserved, centred, not resampled.
+    assert np.array_equal(out[0:13, 2:1022, 2:1022], arr)
+
+
+def test_restore_shape_handles_growth_and_exact_fit():
+    """It must be a no-op when shapes already match, and crop when larger."""
+    from patchworks.plugins.dog import _restore_shape
+
+    same = np.ones((4, 8, 8), dtype="float32")
+    assert _restore_shape(same, (4, 8, 8)).shape == (4, 8, 8)
+    bigger = np.ones((6, 12, 12), dtype="float32")
+    assert _restore_shape(bigger, (4, 8, 8)).shape == (4, 8, 8)
+    # And a mix: one axis short, one long.
+    mixed = np.ones((2, 12), dtype="float32")
+    assert _restore_shape(mixed, (4, 8)).shape == (4, 8)
+
+
+def test_stage_tile_rejects_a_shape_changing_function(tmp_path):
+    """A wrong-shaped return must name the culprit, not blow up inside zarr.
+
+    This used to surface as "could not broadcast input array from shape
+    (13,1020,1020) into shape (14,1024,1024)" six frames deep in zarr's codec
+    pipeline, which says nothing about which function misbehaved.
+    """
+    import dask.array as da
+    import pytest
+
+    from patchworks import create_stage, stage_tile
+
+    image = da.zeros((8, 32, 32), chunks=(4, 16, 16), dtype="uint16")
+    stage = str(tmp_path / "stage.zarr")
+    create_stage(stage, image.shape, (4, 16, 16))
+
+    def crops(block):
+        """Stand-in for a deconvolution backend that trims its output."""
+        return np.zeros(tuple(s - 1 for s in block.shape), dtype="int32")
+
+    with pytest.raises(ValueError, match="one label per input voxel"):
+        stage_tile(image, crops, stage, 0, tile_shape=(4, 16, 16), overlap=2)

{patchworks-2.5.0 → patchworks-2.5.2}/tests/test_notify.py

@@ -73,3 +73,38 @@ def test_log_tail_quotes_the_end_and_survives_a_missing_file(tmp_path):
     assert "could not read" in log_tail(tmp_path / "nope.log")
     (tmp_path / "empty.log").write_text("")
     assert "empty" in log_tail(tmp_path / "empty.log")
+
+
+def test_failing_step_reads_the_rule_from_snakemakes_log(tmp_path):
+    """The failing rule must come from Snakemake, not from log timestamps.
+
+    Guessing by mtime named the wrong step in the most common case: a
+    `segment` failure leaves prepare.log the newest of the sequential steps,
+    so the failure mail quoted a log that had *succeeded*.
+    """
+    from patchworks._notify import failing_step
+
+    log = tmp_path / "sm.log"
+    log.write_text(
+        "Building DAG of jobs...\n"
+        "[Wed Jul 30 11:17:20 2026]\n"
+        "Error in rule segment:\n"
+        "    jobid: 812\n"
+        "    output: seg/203.done\n"
+        "    log: /w/nuclei_labels/logs/segment/203.log (check log file(s))\n"
+    )
+    assert failing_step(log) == (
+        "segment",
+        "/w/nuclei_labels/logs/segment/203.log",
+    )
+
+
+def test_failing_step_degrades_quietly(tmp_path):
+    """It runs inside an error handler, so it must never raise itself."""
+    from patchworks._notify import failing_step
+
+    assert failing_step(None) == (None, None)
+    assert failing_step(tmp_path / "missing.log") == (None, None)
+    clean = tmp_path / "ok.log"
+    clean.write_text("Building DAG of jobs...\n3 of 3 steps (100%) done\n")
+    assert failing_step(clean) == (None, None)

{patchworks-2.5.0 → patchworks-2.5.2}/workflow/Snakefile

@@ -33,20 +33,30 @@ rule all:
 # a local run with no scheduler, and a failure where the useful content is
 # the step's traceback rather than the job's exit code.
 onerror:
-    from patchworks._notify import log_tail, send
+    from patchworks._notify import failing_step, log_tail, send
 
-    _failed = [p for p in (CONVERTLOG, OCCUPANCYLOG, PREPARELOG, MERGELOG)
-               if Path(p).exists()]
-    _newest = max(_failed, key=lambda p: Path(p).stat().st_mtime, default=None)
+    # Snakemake's own log names the failing rule and the log it used. Picking
+    # the most recently written step log instead gets this wrong in the most
+    # common case: a `segment` failure leaves prepare.log newest among the
+    # sequential steps, so the mail quoted a log that had succeeded.
+    _rule, _log = failing_step(log)
+    if _log is None:
+        _cands = [p for p in (CONVERTLOG, OCCUPANCYLOG, PREPARELOG, MERGELOG)
+                  if Path(p).exists()]
+        _cands += [str(p) for p in Path(LOGS).glob("segment/*.log")]
+        _log = max(_cands, key=lambda p: Path(p).stat().st_mtime, default=None)
+    _what = f"rule {_rule}" if _rule else "the workflow"
     send(
         NOTIFY_EMAIL,
-        f"[patchworks] FAILED: {LABEL_NAME} in {WORK}",
-        "The patchworks workflow failed.\n\n"
-        f"  work_dir : {WORK}\n"
-        f"  config   : {LABEL_NAME}\n"
-        f"  log      : {_newest}\n\n"
-        + (f"Last lines of {_newest}:\n\n{log_tail(_newest)}\n"
-           if _newest else "No step log was written.\n"),
+        f"[patchworks] FAILED: {_rule or 'workflow'} - {LABEL_NAME}",
+        f"The patchworks workflow failed in {_what}.\n\n"
+        f"  work_dir      : {WORK}\n"
+        f"  config        : {LABEL_NAME}\n"
+        f"  failing step  : {_rule or 'unknown'}\n"
+        f"  step log      : {_log}\n"
+        f"  snakemake log : {log}\n\n"
+        + (f"Last lines of {_log}:\n\n{log_tail(_log)}\n"
+           if _log else "No step log was written.\n"),
     )
 
 

{patchworks-2.5.0 → patchworks-2.5.2}/workflow/scripts/run_multi.py

@@ -117,6 +117,63 @@ def slurm_jobname_prefix(label: str) -> str:
     return f"pw-{safe}"[:50]
 
 
+def _test_email(cfg: dict) -> int:
+    """Send one test notification and report the outcome. Returns an exit code.
+
+    "No email arrived" has two very different causes that look identical from
+    the outside: the address never made it into the merged config, or it did
+    and the message was dropped somewhere downstream. This distinguishes them
+    without waiting for a multi-hour step to finish.
+    """
+    from patchworks._notify import send, slurm_mail_extra
+
+    email = cfg.get("notify_email") or ""
+    events = cfg.get("notify_events") or ["finish", "error"]
+    if not email:
+        print(
+            "[run_multi] notify_email is empty in the merged config, so no "
+            "mail is sent by design.\n"
+            "  Set it in the file `common:` points at (the per-config files "
+            "no longer carry it), then re-run this check.",
+            file=sys.stderr,
+        )
+        return 1
+
+    print(f"[run_multi] notify_email  : {email}")
+    print(f"[run_multi] notify_events : {events}")
+    print(
+        f"[run_multi] SLURM per-job : sbatch {slurm_mail_extra(email, events)}"
+    )
+    print(
+        "[run_multi] NOTE: convert, occupancy and merge send per-job mail; "
+        "segment does not (one job per tile batch would mean hundreds)."
+    )
+    ok = send(
+        email,
+        "[patchworks] test notification",
+        "This is a patchworks test message.\n\n"
+        "If you received it, the workflow's own success/failure mail will "
+        "reach you too.\n\n"
+        "Per-job start/finish mail is sent by SLURM itself, not by this "
+        "path, so it can still be blocked separately -- verify with:\n"
+        "    scontrol show job <jobid> | grep -i mail\n",
+    )
+    if ok:
+        print(
+            "[run_multi] handed to a local mail transport. If nothing "
+            "arrives, the message was accepted and then dropped further "
+            "along -- ask the cluster admins about outbound mail."
+        )
+        return 0
+    print(
+        "[run_multi] no local mail transport accepted the message (see the "
+        "warning above). SLURM's own per-job mail may still work, since the "
+        "controller sends that, not this host.",
+        file=sys.stderr,
+    )
+    return 1
+
+
 def _run(cmd: list[str], workflow_dir: Path) -> int:
     print(f"[run_multi] $ {' '.join(cmd)}", flush=True)
     return subprocess.run(cmd, cwd=workflow_dir).returncode
@@ -252,6 +309,16 @@ def main() -> None:
             "died: the lock is only released on a clean exit."
         ),
     )
+    parser.add_argument(
+        "--test-email",
+        action="store_true",
+        help=(
+            "send one test message to the configured notify_email and report "
+            "what happened, then exit. Separates 'the address never reached "
+            "the config' from 'the mail was rejected downstream', which "
+            "otherwise look identical: no email either way."
+        ),
+    )
     args = parser.parse_args()
 
     workflow_dir = Path(__file__).resolve().parent.parent
@@ -270,6 +337,9 @@ def main() -> None:
     common_path = _resolve(workflow_dir, common_path) if common_path else None
     common_cfg = _load_yaml(common_path) if common_path else {}
     seg_cfgs = [{**common_cfg, **_load_yaml(p)} for p in seg_config_paths]
+    if args.test_email:
+        sys.exit(_test_email(seg_cfgs[0]))
+
     work_dir = _validate_configs(seg_config_paths, seg_cfgs)
     image_store = f"{work_dir}/image.zarr"
     # Shared by every config, hence keyed on the image and level, not on a