patchworks 2.4.0__tar.gz → 2.5.0__tar.gz

{patchworks-2.4.0 → patchworks-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 2.4.0
+Version: 2.5.0
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-2.4.0 → patchworks-2.5.0}/docs/guide/snakemake.md

@@ -442,6 +442,46 @@ cyto_labels` and `cilia_labels -> nuclei_labels`) so you can use whichever
 fits a given dataset. See `config/config_cilia.yaml`. Its deconvolution step
 needs `pip install "patchworks[dog]"` in the segment jobs' environment.
 
+## Email notifications
+
+Set an address and the workflow mails you when the long steps finish or fail:
+
+```yaml
+# config/common.yaml (or config/config.yaml for a single-config run)
+notify_email: "you@unibas.ch"
+notify_events: ["finish", "error"] # any of: start, finish, error
+```
+
+Leave `notify_email` empty (the default) and nothing is sent.
+
+Per-job mail is **SLURM's own** `--mail-type`, not a message sent from inside
+the job. That matters: the controller sends it, so it still arrives when a job
+is OOM-killed or cancelled by the scheduler — exactly the failures worth
+hearing about, and exactly the ones a notification sent from within the job
+would miss.
+
+It is applied to the long single-job steps only — `convert`, `occupancy` and
+`merge`. `segment` is deliberately excluded: there is one job per tile batch,
+so a thousand-tile run would mean hundreds of messages.
+
+On top of that, the workflow itself sends:
+
+| When | Mail |
+| --- | --- |
+| The run fails | subject `[patchworks] FAILED: <label_name>`, with the last 40 lines of the failing step's log — usually the traceback itself |
+| The run succeeds | subject `[patchworks] done: <label_name>`, with the output label path |
+
+These cover what SLURM cannot: a local run with no scheduler at all, and
+failures where the useful content is the Python traceback rather than an exit
+code.
+
+!!! note "Delivery is best-effort, by design"
+    A notification can never fail a run. If no local `sendmail` exists and no
+    SMTP server answers on localhost, the failure is logged as a warning and
+    the pipeline carries on — a six-hour segmentation that worked must not be
+    reported as failed because a mail host was down. If you get the warning
+    but no mail, ask your cluster admins which relay host to use.
+
 ## Measurements
 
 See [Measurements](measurements.md) for computing area/centroid/intensity

patchworks-2.5.0/src/patchworks/_notify.py

@@ -0,0 +1,157 @@
+"""Best-effort email notification.
+
+Used for the events SLURM cannot report on its own: the end of a whole
+multi-config run, and failures in a local (non-SLURM) run. Per-job
+start/finish mail on a cluster is left to SLURM's own ``--mail-type``, which
+is delivered by the controller and does not depend on a compute node being
+able to reach an MTA.
+
+Every function here is best-effort by design: a notification that cannot be
+delivered must never fail a pipeline that otherwise succeeded, and must never
+turn a real error into a confusing one about email.
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import socket
+import subprocess
+from email.message import EmailMessage
+from pathlib import Path
+from typing import Union
+
+logger = logging.getLogger(__name__)
+
+# Lines of a failing step's log to quote in an error mail. Enough to carry a
+# traceback, short enough that the mail stays readable on a phone.
+LOG_TAIL_LINES = 40
+
+
+def log_tail(path: Union[str, Path], lines: int = LOG_TAIL_LINES) -> str:
+    """Return the last *lines* of a log file, or a note if unreadable."""
+    try:
+        content = Path(path).read_text(errors="replace").splitlines()
+    except OSError as exc:
+        return f"(could not read {path}: {exc})"
+    if not content:
+        return f"({path} is empty)"
+    return "\n".join(content[-lines:])
+
+
+def send(
+    to: Union[str, None],
+    subject: str,
+    body: str,
+    *,
+    sender: Union[str, None] = None,
+) -> bool:
+    """Send *body* to *to*, returning whether it went out.
+
+    Tries a local ``sendmail`` first (what a cluster node normally has), then
+    an SMTP server on localhost. Never raises.
+
+    Parameters
+    ----------
+    to : str or None
+        Recipient. ``None`` or empty disables the notification entirely, which
+        is the default state -- no address configured, no mail, no error.
+    subject : str
+        Subject line.
+    body : str
+        Plain-text body.
+    sender : str, optional
+        From address. Defaults to ``patchworks@<hostname>``.
+
+    Returns
+    -------
+    bool
+        True when the message was handed to a transport.
+    """
+    if not to:
+        return False
+
+    msg = EmailMessage()
+    msg["To"] = to
+    msg["From"] = sender or f"patchworks@{socket.getfqdn()}"
+    msg["Subject"] = subject
+    msg.set_content(body)
+
+    sendmail = shutil.which("sendmail") or shutil.which(
+        "sendmail", path="/usr/sbin:/usr/lib"
+    )
+    if sendmail:
+        try:
+            subprocess.run(
+                [sendmail, "-t", "-oi"],
+                input=msg.as_bytes(),
+                check=True,
+                capture_output=True,
+                timeout=30,
+            )
+            return True
+        except (subprocess.SubprocessError, OSError) as exc:
+            logger.warning(
+                "sendmail failed (%s); trying SMTP on localhost", exc
+            )
+
+    try:
+        import smtplib
+
+        with smtplib.SMTP("localhost", timeout=30) as smtp:
+            smtp.send_message(msg)
+        return True
+    except (OSError, Exception) as exc:  # noqa: BLE001 - never fail the run
+        logger.warning(
+            "could not send notification to %s (%s). The run itself is "
+            "unaffected; set notify_email to '' to silence this.",
+            to,
+            exc,
+        )
+        return False
+
+
+def slurm_mail_extra(
+    email: Union[str, None], events: Union[list, tuple, None] = None
+) -> str:
+    """Build the ``slurm_extra`` fragment for per-job mail.
+
+    SLURM's own ``--mail-type`` is used rather than sending from inside the
+    job: the controller delivers it, so it still arrives when the job is
+    killed by the OOM reaper or the scheduler -- exactly the cases worth
+    hearing about, and exactly the ones an in-job notification misses.
+
+    Parameters
+    ----------
+    email : str or None
+        Recipient; empty/None yields an empty string (no mail configured).
+    events : sequence of str, optional
+        Any of ``"start"``, ``"finish"``, ``"error"``. Defaults to finish and
+        error -- a BEGIN mail per job is rarely worth the inbox.
+
+    Returns
+    -------
+    str
+        Something like ``--mail-type=END,FAIL --mail-user=me@example.org``,
+        or ``""`` when no address is configured.
+
+    Examples
+    --------
+    >>> slurm_mail_extra("me@example.org", ["error"])
+    '--mail-type=FAIL --mail-user=me@example.org'
+    >>> slurm_mail_extra(None)
+    ''
+    """
+    if not email:
+        return ""
+    mapping = {"start": "BEGIN", "finish": "END", "error": "FAIL"}
+    chosen = list(events) if events else ["finish", "error"]
+    unknown = sorted(set(chosen) - set(mapping))
+    if unknown:
+        raise ValueError(
+            f"unknown notify_events {unknown}; use any of "
+            f"{sorted(mapping)} (they map to SLURM's BEGIN/END/FAIL)"
+        )
+    # Keep SLURM's own order, not the config's, so the string is stable.
+    types = [mapping[k] for k in ("start", "finish", "error") if k in chosen]
+    return f"--mail-type={','.join(types)} --mail-user={email}"

patchworks-2.5.0/tests/test_notify.py

@@ -0,0 +1,75 @@
+"""Tests for email notification.
+
+The governing rule: a notification must never be able to fail a run. Delivery
+is best-effort on someone else's mail infrastructure, and a pipeline that
+segmented correctly must not report failure because an SMTP host was down.
+"""
+
+import pytest
+
+from patchworks._notify import log_tail, send, slurm_mail_extra
+
+
+def test_no_address_means_no_mail_and_no_error():
+    """The default state is "unconfigured", which must be completely silent."""
+    assert send(None, "subject", "body") is False
+    assert send("", "subject", "body") is False
+    assert slurm_mail_extra(None) == ""
+    assert slurm_mail_extra("") == ""
+
+
+def test_slurm_mail_extra_maps_events_to_slurm_types():
+    """Config words map to SLURM's BEGIN/END/FAIL, in a stable order."""
+    assert (
+        slurm_mail_extra("me@x.org")
+        == "--mail-type=END,FAIL --mail-user=me@x.org"
+    )
+    assert (
+        slurm_mail_extra("me@x.org", ["error"])
+        == "--mail-type=FAIL --mail-user=me@x.org"
+    )
+    # Order comes from SLURM's lifecycle, not from however the config listed it.
+    assert slurm_mail_extra(
+        "me@x.org", ["error", "start", "finish"]
+    ) == slurm_mail_extra("me@x.org", ["start", "finish", "error"])
+
+
+def test_a_misspelled_event_is_rejected_up_front():
+    """Better a named error before submission than silently no mail.
+
+    The value ends up in an sbatch argument, where a typo would otherwise
+    either be ignored or fail every job at submission time.
+    """
+    with pytest.raises(ValueError, match="notify_events"):
+        slurm_mail_extra("me@x.org", ["finished"])  # not "finish"
+
+
+def test_delivery_failure_is_swallowed(monkeypatch):
+    """An unreachable transport returns False rather than raising.
+
+    This is the property that keeps a mail problem from turning a successful
+    six-hour run into a failed one.
+    """
+    import patchworks._notify as notify
+
+    monkeypatch.setattr(notify.shutil, "which", lambda *a, **k: None)
+
+    def _boom(*args, **kwargs):
+        raise OSError("no route to host")
+
+    monkeypatch.setattr("smtplib.SMTP", _boom)
+    assert send("me@x.org", "subject", "body") is False
+
+
+def test_log_tail_quotes_the_end_and_survives_a_missing_file(tmp_path):
+    """The tail is what makes a failure mail actionable."""
+    p = tmp_path / "step.log"
+    p.write_text("\n".join(f"line {i}" for i in range(200)))
+    tail = log_tail(p, lines=5)
+    assert tail.splitlines() == [f"line {i}" for i in range(195, 200)]
+
+    # A missing or empty log must produce a note, not an exception -- this
+    # runs inside an error handler, where raising would mask the real failure.
+    assert "could not read" in log_tail(tmp_path / "nope.log")
+    (tmp_path / "empty.log").write_text("")
+    assert "empty" in log_tail(tmp_path / "empty.log")

patchworks-2.5.0/workflow/Snakefile

@@ -0,0 +1,64 @@
+# patchworks Snakemake workflow.
+#
+#   convert ──▶ prepare (checkpoint) ──▶ segment {tile}  ──▶ merge
+#                                        one GPU job per tile
+#
+# Run locally:
+#   snakemake --cores 8 --configfile config/config.yaml
+# Run on SLURM (one GPU job per tile, many GPUs in parallel):
+#   snakemake --workflow-profile profile/slurm --configfile config/config.yaml
+
+import json
+from pathlib import Path
+
+configfile: "config/config.yaml"
+
+include: "rules/common.smk"
+include: "rules/convert.smk"
+include: "rules/segment.smk"
+include: "rules/merge.smk"
+
+
+# Runs on the submit host (has network) — never submitted to an offline GPU node.
+localrules:
+    fetch_model,
+
+
+rule all:
+    input:
+        f"{RUN}/labels.done",
+
+
+# Workflow-level notifications, covering what SLURM's per-job mail cannot:
+# a local run with no scheduler, and a failure where the useful content is
+# the step's traceback rather than the job's exit code.
+onerror:
+    from patchworks._notify import log_tail, send
+
+    _failed = [p for p in (CONVERTLOG, OCCUPANCYLOG, PREPARELOG, MERGELOG)
+               if Path(p).exists()]
+    _newest = max(_failed, key=lambda p: Path(p).stat().st_mtime, default=None)
+    send(
+        NOTIFY_EMAIL,
+        f"[patchworks] FAILED: {LABEL_NAME} in {WORK}",
+        "The patchworks workflow failed.\n\n"
+        f"  work_dir : {WORK}\n"
+        f"  config   : {LABEL_NAME}\n"
+        f"  log      : {_newest}\n\n"
+        + (f"Last lines of {_newest}:\n\n{log_tail(_newest)}\n"
+           if _newest else "No step log was written.\n"),
+    )
+
+
+onsuccess:
+    if "finish" in NOTIFY_EVENTS:
+        from patchworks._notify import send
+
+        send(
+            NOTIFY_EMAIL,
+            f"[patchworks] done: {LABEL_NAME} in {WORK}",
+            f"The patchworks workflow finished successfully.\n\n"
+            f"  work_dir : {WORK}\n"
+            f"  config   : {LABEL_NAME}\n"
+            f"  labels   : {IMAGE}/labels/{LABEL_NAME}\n",
+        )

{patchworks-2.4.0 → patchworks-2.5.0}/workflow/config/common.yaml

@@ -40,3 +40,12 @@ tiles_per_job: 4
 # --- merge ---
 sequential_labels: true
 merge_workers: null
+
+# --- notifications (optional) ---
+# Leave empty for no email. Per-job mail is sent by SLURM itself for the long
+# single-job steps (convert, occupancy, merge), so it still arrives when a job
+# is OOM-killed. `segment` is deliberately excluded: one job per tile batch
+# would mean hundreds of messages.
+notify_email: ""
+# Any of: start, finish, error  (SLURM BEGIN, END, FAIL)
+notify_events: ["finish", "error"]

{patchworks-2.4.0 → patchworks-2.5.0}/workflow/config/config.yaml

@@ -80,3 +80,8 @@ sequential_labels: true # renumber merged labels to a contiguous 1..N range
 # limit and cpus_per_task, not the node's totals. The merge logs what it
 # detected, so you can compare that line against `seff <jobid>`.
 merge_workers: null
+
+# --- notifications (optional) ---
+# Email on step completion/failure. Empty = off. See docs/guide/snakemake.md.
+notify_email: ""
+notify_events: ["finish", "error"]

{patchworks-2.4.0 → patchworks-2.5.0}/workflow/rules/common.smk

@@ -36,7 +36,9 @@ STAGE_OK = f"{STAGE}.done"
 # wiped the previous one's output -- by the time a run finished, only the last
 # step's log survived and a failure earlier on left nothing to read.
 LOGS = f"{RUN}/logs"
-CONVERTLOG = f"{LOGS}/convert.log"
+# convert and occupancy are shared by every config, so their logs belong
+# beside the image, not under whichever config happened to run phase A.
+CONVERTLOG = f"{WORK}/logs/convert.log"
 PREPARELOG = f"{LOGS}/prepare.log"
 MERGELOG = f"{LOGS}/merge.log"
 
@@ -57,3 +59,22 @@ def batch_done(wildcards):
     tiles = checkpoints.prepare.get().output.tiles
     manifest = json.loads(Path(tiles).read_text())
     return [f"{RUN}/seg/{i}.done" for i in range(len(manifest["batches"]))]
+
+
+# --- notifications -----------------------------------------------------
+# Per-job mail is SLURM's own --mail-type: the controller sends it, so it
+# still arrives when a job is OOM-killed or cancelled by the scheduler --
+# the cases most worth hearing about, and the ones a notification sent from
+# inside the job would miss. Empty when no address is configured.
+#
+# Deliberately NOT applied to `segment`: there is one job per tile batch, so
+# a thousand-tile run would mean hundreds of emails.
+NOTIFY_EMAIL = config.get("notify_email") or ""
+NOTIFY_EVENTS = config.get("notify_events") or ["finish", "error"]
+
+
+def notify_extra(*_args, **_kwargs):
+    """slurm_extra fragment requesting mail for this job."""
+    from patchworks._notify import slurm_mail_extra
+
+    return slurm_mail_extra(NOTIFY_EMAIL, NOTIFY_EVENTS)

{patchworks-2.4.0 → patchworks-2.5.0}/workflow/rules/convert.smk

@@ -4,6 +4,8 @@ rule convert:
     output:
         # marker file inside the store; existence => skip re-conversion.
         IMAGE_OK,
+    resources:
+        slurm_extra=notify_extra,
     log:
         CONVERTLOG,
     script:
@@ -18,6 +20,8 @@ rule occupancy:
         IMAGE_OK,
     output:
         OCCUPANCY_OK,
+    resources:
+        slurm_extra=notify_extra,
     log:
         OCCUPANCYLOG,
     script:

{patchworks-2.4.0 → patchworks-2.5.0}/workflow/rules/merge.smk

@@ -6,6 +6,8 @@ rule merge:
         tiles=TILES,
     output:
         touch(f"{RUN}/labels.done"),
+    resources:
+        slurm_extra=notify_extra,
     log:
         MERGELOG,
     script:

patchworks-2.4.0/workflow/Snakefile

@@ -1,29 +0,0 @@
-# patchworks Snakemake workflow.
-#
-#   convert ──▶ prepare (checkpoint) ──▶ segment {tile}  ──▶ merge
-#                                        one GPU job per tile
-#
-# Run locally:
-#   snakemake --cores 8 --configfile config/config.yaml
-# Run on SLURM (one GPU job per tile, many GPUs in parallel):
-#   snakemake --workflow-profile profile/slurm --configfile config/config.yaml
-
-import json
-from pathlib import Path
-
-configfile: "config/config.yaml"
-
-include: "rules/common.smk"
-include: "rules/convert.smk"
-include: "rules/segment.smk"
-include: "rules/merge.smk"
-
-
-# Runs on the submit host (has network) — never submitted to an offline GPU node.
-localrules:
-    fetch_model,
-
-
-rule all:
-    input:
-        f"{RUN}/labels.done",