patchworks 2.4.0__tar.gz → 2.5.1__tar.gz

{patchworks-2.4.0 → patchworks-2.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 2.4.0
+Version: 2.5.1
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-2.4.0 → patchworks-2.5.1}/docs/guide/snakemake.md

@@ -442,6 +442,46 @@ cyto_labels` and `cilia_labels -> nuclei_labels`) so you can use whichever
 fits a given dataset. See `config/config_cilia.yaml`. Its deconvolution step
 needs `pip install "patchworks[dog]"` in the segment jobs' environment.
 
+## Email notifications
+
+Set an address and the workflow mails you when the long steps finish or fail:
+
+```yaml
+# config/common.yaml (or config/config.yaml for a single-config run)
+notify_email: "you@unibas.ch"
+notify_events: ["finish", "error"] # any of: start, finish, error
+```
+
+Leave `notify_email` empty (the default) and nothing is sent.
+
+Per-job mail is **SLURM's own** `--mail-type`, not a message sent from inside
+the job. That matters: the controller sends it, so it still arrives when a job
+is OOM-killed or cancelled by the scheduler — exactly the failures worth
+hearing about, and exactly the ones a notification sent from within the job
+would miss.
+
+It is applied to the long single-job steps only — `convert`, `occupancy` and
+`merge`. `segment` is deliberately excluded: there is one job per tile batch,
+so a thousand-tile run would mean hundreds of messages.
+
+On top of that, the workflow itself sends:
+
+| When | Mail |
+| --- | --- |
+| The run fails | subject `[patchworks] FAILED: <label_name>`, with the last 40 lines of the failing step's log — usually the traceback itself |
+| The run succeeds | subject `[patchworks] done: <label_name>`, with the output label path |
+
+These cover what SLURM cannot: a local run with no scheduler at all, and
+failures where the useful content is the Python traceback rather than an exit
+code.
+
+!!! note "Delivery is best-effort, by design"
+    A notification can never fail a run. If no local `sendmail` exists and no
+    SMTP server answers on localhost, the failure is logged as a warning and
+    the pipeline carries on — a six-hour segmentation that worked must not be
+    reported as failed because a mail host was down. If you get the warning
+    but no mail, ask your cluster admins which relay host to use.
+
 ## Measurements
 
 See [Measurements](measurements.md) for computing area/centroid/intensity

{patchworks-2.4.0 → patchworks-2.5.1}/src/patchworks/_distributed.py

@@ -213,6 +213,22 @@ def stage_tile(
         trims.append((s.start - lo, hi - s.stop))
     block = np.asarray(image[tuple(expanded)])
     out = np.asarray(fn(block))
+    if out.shape != block.shape:
+        # Caught here rather than 6 frames deep in zarr's codec pipeline as
+        # "could not broadcast input array from shape (13,1020,1020) into
+        # shape (14,1024,1024)", which says nothing about which function is
+        # at fault. A segmentation function must label the voxels it was
+        # given: the halo trim and the destination slice are both computed
+        # from the tile's geometry, so a different shape has no defined
+        # placement.
+        name = getattr(fn, "__name__", type(fn).__name__)
+        raise ValueError(
+            f"segmentation function {name!r} returned shape {out.shape} for "
+            f"a tile of shape {block.shape} (tile {index}). It must return "
+            "one label per input voxel. Some deconvolution backends crop "
+            "their output -- pad or centre it back to the input shape before "
+            "returning."
+        )
     sel = tuple(
         slice(left, out.shape[i] - right)
         for i, (left, right) in enumerate(trims)

patchworks-2.5.1/src/patchworks/_notify.py

@@ -0,0 +1,157 @@
+"""Best-effort email notification.
+
+Used for the events SLURM cannot report on its own: the end of a whole
+multi-config run, and failures in a local (non-SLURM) run. Per-job
+start/finish mail on a cluster is left to SLURM's own ``--mail-type``, which
+is delivered by the controller and does not depend on a compute node being
+able to reach an MTA.
+
+Every function here is best-effort by design: a notification that cannot be
+delivered must never fail a pipeline that otherwise succeeded, and must never
+turn a real error into a confusing one about email.
+"""
+
+from __future__ import annotations
+
+import logging
+import shutil
+import socket
+import subprocess
+from email.message import EmailMessage
+from pathlib import Path
+from typing import Union
+
+logger = logging.getLogger(__name__)
+
+# Lines of a failing step's log to quote in an error mail. Enough to carry a
+# traceback, short enough that the mail stays readable on a phone.
+LOG_TAIL_LINES = 40
+
+
+def log_tail(path: Union[str, Path], lines: int = LOG_TAIL_LINES) -> str:
+    """Return the last *lines* of a log file, or a note if unreadable."""
+    try:
+        content = Path(path).read_text(errors="replace").splitlines()
+    except OSError as exc:
+        return f"(could not read {path}: {exc})"
+    if not content:
+        return f"({path} is empty)"
+    return "\n".join(content[-lines:])
+
+
+def send(
+    to: Union[str, None],
+    subject: str,
+    body: str,
+    *,
+    sender: Union[str, None] = None,
+) -> bool:
+    """Send *body* to *to*, returning whether it went out.
+
+    Tries a local ``sendmail`` first (what a cluster node normally has), then
+    an SMTP server on localhost. Never raises.
+
+    Parameters
+    ----------
+    to : str or None
+        Recipient. ``None`` or empty disables the notification entirely, which
+        is the default state -- no address configured, no mail, no error.
+    subject : str
+        Subject line.
+    body : str
+        Plain-text body.
+    sender : str, optional
+        From address. Defaults to ``patchworks@<hostname>``.
+
+    Returns
+    -------
+    bool
+        True when the message was handed to a transport.
+    """
+    if not to:
+        return False
+
+    msg = EmailMessage()
+    msg["To"] = to
+    msg["From"] = sender or f"patchworks@{socket.getfqdn()}"
+    msg["Subject"] = subject
+    msg.set_content(body)
+
+    sendmail = shutil.which("sendmail") or shutil.which(
+        "sendmail", path="/usr/sbin:/usr/lib"
+    )
+    if sendmail:
+        try:
+            subprocess.run(
+                [sendmail, "-t", "-oi"],
+                input=msg.as_bytes(),
+                check=True,
+                capture_output=True,
+                timeout=30,
+            )
+            return True
+        except (subprocess.SubprocessError, OSError) as exc:
+            logger.warning(
+                "sendmail failed (%s); trying SMTP on localhost", exc
+            )
+
+    try:
+        import smtplib
+
+        with smtplib.SMTP("localhost", timeout=30) as smtp:
+            smtp.send_message(msg)
+        return True
+    except (OSError, Exception) as exc:  # noqa: BLE001 - never fail the run
+        logger.warning(
+            "could not send notification to %s (%s). The run itself is "
+            "unaffected; set notify_email to '' to silence this.",
+            to,
+            exc,
+        )
+        return False
+
+
+def slurm_mail_extra(
+    email: Union[str, None], events: Union[list, tuple, None] = None
+) -> str:
+    """Build the ``slurm_extra`` fragment for per-job mail.
+
+    SLURM's own ``--mail-type`` is used rather than sending from inside the
+    job: the controller delivers it, so it still arrives when the job is
+    killed by the OOM reaper or the scheduler -- exactly the cases worth
+    hearing about, and exactly the ones an in-job notification misses.
+
+    Parameters
+    ----------
+    email : str or None
+        Recipient; empty/None yields an empty string (no mail configured).
+    events : sequence of str, optional
+        Any of ``"start"``, ``"finish"``, ``"error"``. Defaults to finish and
+        error -- a BEGIN mail per job is rarely worth the inbox.
+
+    Returns
+    -------
+    str
+        Something like ``--mail-type=END,FAIL --mail-user=me@example.org``,
+        or ``""`` when no address is configured.
+
+    Examples
+    --------
+    >>> slurm_mail_extra("me@example.org", ["error"])
+    '--mail-type=FAIL --mail-user=me@example.org'
+    >>> slurm_mail_extra(None)
+    ''
+    """
+    if not email:
+        return ""
+    mapping = {"start": "BEGIN", "finish": "END", "error": "FAIL"}
+    chosen = list(events) if events else ["finish", "error"]
+    unknown = sorted(set(chosen) - set(mapping))
+    if unknown:
+        raise ValueError(
+            f"unknown notify_events {unknown}; use any of "
+            f"{sorted(mapping)} (they map to SLURM's BEGIN/END/FAIL)"
+        )
+    # Keep SLURM's own order, not the config's, so the string is stable.
+    types = [mapping[k] for k in ("start", "finish", "error") if k in chosen]
+    return f"--mail-type={','.join(types)} --mail-user={email}"

{patchworks-2.4.0 → patchworks-2.5.1}/src/patchworks/plugins/dog.py

@@ -238,6 +238,61 @@ def _run(block: np.ndarray, dog_dict: dict[str, Any]) -> np.ndarray:
     )
 
 
+def _restore_shape(arr: np.ndarray, shape: tuple[int, ...]) -> np.ndarray:
+    """Centre *arr* back into an array of *shape*, cropping or edge-padding.
+
+    Deconvolution must not change the field of view: patchworks writes the
+    result into a destination slice derived from the tile's geometry, so one
+    label per input voxel is required.
+
+    Centring is the right correction for a symmetric crop, which is what
+    apodisation produces. The discrepancies observed are small (a voxel in z,
+    a few in x/y) and land inside the halo, which is discarded anyway -- so
+    the labels that survive the trim are unaffected. It is logged at WARNING
+    with the exact shapes so a larger, non-symmetric crop cannot pass
+    silently.
+
+    Parameters
+    ----------
+    arr : np.ndarray
+        The deconvolved volume.
+    shape : tuple of int
+        The shape it must be returned at (the input tile's).
+
+    Returns
+    -------
+    np.ndarray
+        An array of exactly *shape*.
+
+    Examples
+    --------
+    >>> import numpy as np
+    >>> _restore_shape(np.ones((13, 1020)), (14, 1024)).shape
+    (14, 1024)
+    """
+    logger.warning(
+        "deconvolution returned %s for a %s input; re-centring to the input "
+        "shape. patchworks needs one label per input voxel. A large or "
+        "asymmetric difference here would shift labels -- check the PSF and "
+        "voxel sizes if this is more than a few voxels.",
+        arr.shape,
+        shape,
+    )
+    # Crop first, so an axis that grew is handled before padding the rest.
+    crop = tuple(
+        slice((a - s) // 2, (a - s) // 2 + s) if a > s else slice(None)
+        for a, s in zip(arr.shape, shape)
+    )
+    arr = arr[crop]
+    pad = tuple(
+        ((s - a) // 2, s - a - (s - a) // 2) if a < s else (0, 0)
+        for a, s in zip(arr.shape, shape)
+    )
+    if any(lo or hi for lo, hi in pad):
+        arr = np.pad(arr, pad, mode="edge")
+    return arr
+
+
 def _segment_once(
     block: np.ndarray, dog_dict: dict[str, Any], use_gpu: bool
 ) -> np.ndarray:
@@ -253,7 +308,16 @@ def _segment_once(
         # in the per-tile timing that tile_process logs.
         from pycudadecon import decon
 
+        before = img.shape
         img = decon(images=img, **decon_kwargs)
+        if img.shape != before:
+            # cudaDecon returns a slightly smaller volume for some input
+            # sizes (e.g. (14,1024,1024) -> (13,1020,1020) on an edge tile).
+            # patchworks needs one label per input voxel: the halo trim and
+            # the destination slice are both derived from the tile geometry,
+            # so a shrunken result has nowhere to go and used to surface as
+            # an unreadable broadcast error from inside zarr.
+            img = _restore_shape(img, before)
 
     if use_gpu:
         import cupy as cp

{patchworks-2.4.0 → patchworks-2.5.1}/tests/test_dog.py

@@ -107,3 +107,56 @@ def test_explicit_decon_kwargs_win_over_the_calibration(monkeypatch):
     assert captured["dxdata"] == 0.1  # filled from the calibration
     assert captured["dzdata"] == 0.2
     assert captured["dzpsf"] == 0.2
+
+
+def test_restore_shape_recentres_a_cropped_decon():
+    """cudaDecon can hand back a smaller volume than it was given.
+
+    Observed on a real edge tile: (14, 1024, 1024) in, (13, 1020, 1020) out.
+    patchworks needs one label per input voxel, so the field of view has to be
+    restored before the DoG step.
+    """
+    from patchworks.plugins.dog import _restore_shape
+
+    arr = np.arange(13 * 1020 * 1020, dtype="float32").reshape(13, 1020, 1020)
+    out = _restore_shape(arr, (14, 1024, 1024))
+    assert out.shape == (14, 1024, 1024)
+    # The original content is preserved, centred, not resampled.
+    assert np.array_equal(out[0:13, 2:1022, 2:1022], arr)
+
+
+def test_restore_shape_handles_growth_and_exact_fit():
+    """It must be a no-op when shapes already match, and crop when larger."""
+    from patchworks.plugins.dog import _restore_shape
+
+    same = np.ones((4, 8, 8), dtype="float32")
+    assert _restore_shape(same, (4, 8, 8)).shape == (4, 8, 8)
+    bigger = np.ones((6, 12, 12), dtype="float32")
+    assert _restore_shape(bigger, (4, 8, 8)).shape == (4, 8, 8)
+    # And a mix: one axis short, one long.
+    mixed = np.ones((2, 12), dtype="float32")
+    assert _restore_shape(mixed, (4, 8)).shape == (4, 8)
+
+
+def test_stage_tile_rejects_a_shape_changing_function(tmp_path):
+    """A wrong-shaped return must name the culprit, not blow up inside zarr.
+
+    This used to surface as "could not broadcast input array from shape
+    (13,1020,1020) into shape (14,1024,1024)" six frames deep in zarr's codec
+    pipeline, which says nothing about which function misbehaved.
+    """
+    import dask.array as da
+    import pytest
+
+    from patchworks import create_stage, stage_tile
+
+    image = da.zeros((8, 32, 32), chunks=(4, 16, 16), dtype="uint16")
+    stage = str(tmp_path / "stage.zarr")
+    create_stage(stage, image.shape, (4, 16, 16))
+
+    def crops(block):
+        """Stand-in for a deconvolution backend that trims its output."""
+        return np.zeros(tuple(s - 1 for s in block.shape), dtype="int32")
+
+    with pytest.raises(ValueError, match="one label per input voxel"):
+        stage_tile(image, crops, stage, 0, tile_shape=(4, 16, 16), overlap=2)

patchworks-2.5.1/tests/test_notify.py

@@ -0,0 +1,75 @@
+"""Tests for email notification.
+
+The governing rule: a notification must never be able to fail a run. Delivery
+is best-effort on someone else's mail infrastructure, and a pipeline that
+segmented correctly must not report failure because an SMTP host was down.
+"""
+
+import pytest
+
+from patchworks._notify import log_tail, send, slurm_mail_extra
+
+
+def test_no_address_means_no_mail_and_no_error():
+    """The default state is "unconfigured", which must be completely silent."""
+    assert send(None, "subject", "body") is False
+    assert send("", "subject", "body") is False
+    assert slurm_mail_extra(None) == ""
+    assert slurm_mail_extra("") == ""
+
+
+def test_slurm_mail_extra_maps_events_to_slurm_types():
+    """Config words map to SLURM's BEGIN/END/FAIL, in a stable order."""
+    assert (
+        slurm_mail_extra("me@x.org")
+        == "--mail-type=END,FAIL --mail-user=me@x.org"
+    )
+    assert (
+        slurm_mail_extra("me@x.org", ["error"])
+        == "--mail-type=FAIL --mail-user=me@x.org"
+    )
+    # Order comes from SLURM's lifecycle, not from however the config listed it.
+    assert slurm_mail_extra(
+        "me@x.org", ["error", "start", "finish"]
+    ) == slurm_mail_extra("me@x.org", ["start", "finish", "error"])
+
+
+def test_a_misspelled_event_is_rejected_up_front():
+    """Better a named error before submission than silently no mail.
+
+    The value ends up in an sbatch argument, where a typo would otherwise
+    either be ignored or fail every job at submission time.
+    """
+    with pytest.raises(ValueError, match="notify_events"):
+        slurm_mail_extra("me@x.org", ["finished"])  # not "finish"
+
+
+def test_delivery_failure_is_swallowed(monkeypatch):
+    """An unreachable transport returns False rather than raising.
+
+    This is the property that keeps a mail problem from turning a successful
+    six-hour run into a failed one.
+    """
+    import patchworks._notify as notify
+
+    monkeypatch.setattr(notify.shutil, "which", lambda *a, **k: None)
+
+    def _boom(*args, **kwargs):
+        raise OSError("no route to host")
+
+    monkeypatch.setattr("smtplib.SMTP", _boom)
+    assert send("me@x.org", "subject", "body") is False
+
+
+def test_log_tail_quotes_the_end_and_survives_a_missing_file(tmp_path):
+    """The tail is what makes a failure mail actionable."""
+    p = tmp_path / "step.log"
+    p.write_text("\n".join(f"line {i}" for i in range(200)))
+    tail = log_tail(p, lines=5)
+    assert tail.splitlines() == [f"line {i}" for i in range(195, 200)]
+
+    # A missing or empty log must produce a note, not an exception -- this
+    # runs inside an error handler, where raising would mask the real failure.
+    assert "could not read" in log_tail(tmp_path / "nope.log")
+    (tmp_path / "empty.log").write_text("")
+    assert "empty" in log_tail(tmp_path / "empty.log")

patchworks-2.5.1/workflow/Snakefile

@@ -0,0 +1,64 @@
+# patchworks Snakemake workflow.
+#
+#   convert ──▶ prepare (checkpoint) ──▶ segment {tile}  ──▶ merge
+#                                        one GPU job per tile
+#
+# Run locally:
+#   snakemake --cores 8 --configfile config/config.yaml
+# Run on SLURM (one GPU job per tile, many GPUs in parallel):
+#   snakemake --workflow-profile profile/slurm --configfile config/config.yaml
+
+import json
+from pathlib import Path
+
+configfile: "config/config.yaml"
+
+include: "rules/common.smk"
+include: "rules/convert.smk"
+include: "rules/segment.smk"
+include: "rules/merge.smk"
+
+
+# Runs on the submit host (has network) — never submitted to an offline GPU node.
+localrules:
+    fetch_model,
+
+
+rule all:
+    input:
+        f"{RUN}/labels.done",
+
+
+# Workflow-level notifications, covering what SLURM's per-job mail cannot:
+# a local run with no scheduler, and a failure where the useful content is
+# the step's traceback rather than the job's exit code.
+onerror:
+    from patchworks._notify import log_tail, send
+
+    _failed = [p for p in (CONVERTLOG, OCCUPANCYLOG, PREPARELOG, MERGELOG)
+               if Path(p).exists()]
+    _newest = max(_failed, key=lambda p: Path(p).stat().st_mtime, default=None)
+    send(
+        NOTIFY_EMAIL,
+        f"[patchworks] FAILED: {LABEL_NAME} in {WORK}",
+        "The patchworks workflow failed.\n\n"
+        f"  work_dir : {WORK}\n"
+        f"  config   : {LABEL_NAME}\n"
+        f"  log      : {_newest}\n\n"
+        + (f"Last lines of {_newest}:\n\n{log_tail(_newest)}\n"
+           if _newest else "No step log was written.\n"),
+    )
+
+
+onsuccess:
+    if "finish" in NOTIFY_EVENTS:
+        from patchworks._notify import send
+
+        send(
+            NOTIFY_EMAIL,
+            f"[patchworks] done: {LABEL_NAME} in {WORK}",
+            f"The patchworks workflow finished successfully.\n\n"
+            f"  work_dir : {WORK}\n"
+            f"  config   : {LABEL_NAME}\n"
+            f"  labels   : {IMAGE}/labels/{LABEL_NAME}\n",
+        )

{patchworks-2.4.0 → patchworks-2.5.1}/workflow/config/common.yaml

@@ -40,3 +40,12 @@ tiles_per_job: 4
 # --- merge ---
 sequential_labels: true
 merge_workers: null
+
+# --- notifications (optional) ---
+# Leave empty for no email. Per-job mail is sent by SLURM itself for the long
+# single-job steps (convert, occupancy, merge), so it still arrives when a job
+# is OOM-killed. `segment` is deliberately excluded: one job per tile batch
+# would mean hundreds of messages.
+notify_email: ""
+# Any of: start, finish, error  (SLURM BEGIN, END, FAIL)
+notify_events: ["finish", "error"]

{patchworks-2.4.0 → patchworks-2.5.1}/workflow/config/config.yaml

@@ -80,3 +80,8 @@ sequential_labels: true # renumber merged labels to a contiguous 1..N range
 # limit and cpus_per_task, not the node's totals. The merge logs what it
 # detected, so you can compare that line against `seff <jobid>`.
 merge_workers: null
+
+# --- notifications (optional) ---
+# Email on step completion/failure. Empty = off. See docs/guide/snakemake.md.
+notify_email: ""
+notify_events: ["finish", "error"]

{patchworks-2.4.0 → patchworks-2.5.1}/workflow/rules/common.smk

@@ -36,7 +36,9 @@ STAGE_OK = f"{STAGE}.done"
 # wiped the previous one's output -- by the time a run finished, only the last
 # step's log survived and a failure earlier on left nothing to read.
 LOGS = f"{RUN}/logs"
-CONVERTLOG = f"{LOGS}/convert.log"
+# convert and occupancy are shared by every config, so their logs belong
+# beside the image, not under whichever config happened to run phase A.
+CONVERTLOG = f"{WORK}/logs/convert.log"
 PREPARELOG = f"{LOGS}/prepare.log"
 MERGELOG = f"{LOGS}/merge.log"
 
@@ -57,3 +59,22 @@ def batch_done(wildcards):
     tiles = checkpoints.prepare.get().output.tiles
     manifest = json.loads(Path(tiles).read_text())
     return [f"{RUN}/seg/{i}.done" for i in range(len(manifest["batches"]))]
+
+
+# --- notifications -----------------------------------------------------
+# Per-job mail is SLURM's own --mail-type: the controller sends it, so it
+# still arrives when a job is OOM-killed or cancelled by the scheduler --
+# the cases most worth hearing about, and the ones a notification sent from
+# inside the job would miss. Empty when no address is configured.
+#
+# Deliberately NOT applied to `segment`: there is one job per tile batch, so
+# a thousand-tile run would mean hundreds of emails.
+NOTIFY_EMAIL = config.get("notify_email") or ""
+NOTIFY_EVENTS = config.get("notify_events") or ["finish", "error"]
+
+
+def notify_extra(*_args, **_kwargs):
+    """slurm_extra fragment requesting mail for this job."""
+    from patchworks._notify import slurm_mail_extra
+
+    return slurm_mail_extra(NOTIFY_EMAIL, NOTIFY_EVENTS)

{patchworks-2.4.0 → patchworks-2.5.1}/workflow/rules/convert.smk

@@ -4,6 +4,8 @@ rule convert:
     output:
         # marker file inside the store; existence => skip re-conversion.
         IMAGE_OK,
+    resources:
+        slurm_extra=notify_extra,
     log:
         CONVERTLOG,
     script:
@@ -18,6 +20,8 @@ rule occupancy:
         IMAGE_OK,
     output:
         OCCUPANCY_OK,
+    resources:
+        slurm_extra=notify_extra,
     log:
         OCCUPANCYLOG,
     script:

{patchworks-2.4.0 → patchworks-2.5.1}/workflow/rules/merge.smk

@@ -6,6 +6,8 @@ rule merge:
         tiles=TILES,
     output:
         touch(f"{RUN}/labels.done"),
+    resources:
+        slurm_extra=notify_extra,
     log:
         MERGELOG,
     script:

patchworks-2.4.0/workflow/Snakefile

@@ -1,29 +0,0 @@
-# patchworks Snakemake workflow.
-#
-#   convert ──▶ prepare (checkpoint) ──▶ segment {tile}  ──▶ merge
-#                                        one GPU job per tile
-#
-# Run locally:
-#   snakemake --cores 8 --configfile config/config.yaml
-# Run on SLURM (one GPU job per tile, many GPUs in parallel):
-#   snakemake --workflow-profile profile/slurm --configfile config/config.yaml
-
-import json
-from pathlib import Path
-
-configfile: "config/config.yaml"
-
-include: "rules/common.smk"
-include: "rules/convert.smk"
-include: "rules/segment.smk"
-include: "rules/merge.smk"
-
-
-# Runs on the submit host (has network) — never submitted to an offline GPU node.
-localrules:
-    fetch_model,
-
-
-rule all:
-    input:
-        f"{RUN}/labels.done",