patchworks 0.11.5__tar.gz → 0.11.7__tar.gz

{patchworks-0.11.5 → patchworks-0.11.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: patchworks
-Version: 0.11.5
+Version: 0.11.7
 Summary: Tiled processing of arbitrarily large images with globally consistent labels
 Project-URL: Homepage, https://github.com/imcf/patchworks
 Project-URL: Issues, https://github.com/imcf/patchworks/issues

{patchworks-0.11.5 → patchworks-0.11.7}/docs/guide/snakemake.md

@@ -223,10 +223,12 @@ prologue. The simplest path is a single shared env that the compute nodes see.
 | Symptom | Fix |
 |---------|-----|
 | `snakemake: command not found` | use `python -m snakemake` |
-| Segment jobs pend forever | wrong `slurm_partition`/`slurm_extra` GPU flag for your cluster |
+| Segment jobs pend forever | wrong `slurm_partition`/GPU request; on scicore use `gres: "gpu:1"` |
+| Segment dies, `Network is unreachable` | offline GPU nodes — the `fetch_model` localrule caches the model on the submit host first; if it still fails, your submit host has no network either (pre-download manually) |
 | `cellpose is not installed` in a job | the job's env lacks `patchworks[cellpose]` |
 | Reading the input fails | install the matching reader (`patchworks[imaris]`/`[bioio]` + a `bioio-*`) |
 | Out of GPU memory | smaller `tile_shape`, or `do_3D: false` |
+| A job fails with an empty SLURM log | read `logs/segment/<index>.log` (per tile) or `logs/steps.log` — the real traceback is there |
 | Very slow | confirm GPU is used (`nvidia-smi`); try 2-D or a lower `level` |
 
 ## How it works (for the curious)

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/Snakefile

@@ -19,6 +19,11 @@ include: "rules/segment.smk"
 include: "rules/merge.smk"
 
 
+# Runs on the submit host (has network) — never submitted to an offline GPU node.
+localrules:
+    fetch_model,
+
+
 rule all:
     input:
         f"{WORK}/labels.done",

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/pixi.toml

@@ -7,7 +7,7 @@
 # On a cluster, put this project on a shared filesystem the compute nodes can
 # read: the SLURM jobs re-launch snakemake from this env's interpreter.
 
-[project]
+[workspace]
 name = "patchworks-workflow"
 channels = ["conda-forge"]
 platforms = ["linux-64"]

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/profile/slurm/config.yaml

@@ -17,7 +17,7 @@ keep-going: true
 
 # Defaults for every rule (CPU jobs).
 default-resources:
-  slurm_partition: "cpu"
+  slurm_partition: "scicore"
   # slurm_account: "my_account"
   mem_mb: 16000
   cpus_per_task: 4
@@ -32,12 +32,14 @@ set-resources:
   prepare:
     mem_mb: 32000
     runtime: 120
-  segment: # one GPU per tile — this is what spreads Cellpose across GPUs
-    slurm_partition: "gpu"
-    slurm_extra: "'--gres=gpu:1'"
-    mem_mb: 32000
+  segment:
+    # one GPU per tile — this is what spreads Cellpose across GPUs
+    slurm_partition: "rtx4090"
+    gpu: 1
+    qos: "rtx4090-6hours" # scicore: <partition>-<duration> QOS
+    mem_mb: 32000 # plenty — a tile used ~1G; the failure was a Python error
     cpus_per_task: 4
-    runtime: 120
+    runtime: 360 # 6 hours — must match the QOS, NOT 120 (=2h → killed early)
   merge:
     mem_mb: 128000
     cpus_per_task: 8

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/rules/common.smk

@@ -15,6 +15,16 @@ STAGE = f"{WORK}/stage.zarr"
 STAGE_OK = f"{STAGE}.done"
 
 
+# Logs: one shared file for the sequential CPU steps (convert/prepare/merge),
+# one file per tile for the GPU segment jobs.
+LOGS = f"{WORK}/logs"
+STEPLOG = f"{LOGS}/steps.log"
+
+# Marker that the segmentation model is cached locally. Produced by a local
+# rule (runs on the networked submit host) so offline GPU nodes never download.
+MODEL_OK = f"{WORK}/model.ready"
+
+
 def occupied_done(wildcards):
     """Per-tile markers for the occupied tiles (resolved after the checkpoint)."""
     tiles = checkpoints.prepare.get().output.tiles

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/rules/convert.smk

@@ -4,5 +4,7 @@ rule convert:
     output:
         # marker file inside the store; existence => skip re-conversion.
         IMAGE_OK,
+    log:
+        STEPLOG,
     script:
         "../scripts/convert.py"

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/rules/merge.smk

@@ -5,5 +5,7 @@ rule merge:
         occupied_done,
     output:
         touch(f"{WORK}/labels.done"),
+    log:
+        STEPLOG,
     script:
         "../scripts/merge.py"

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/rules/segment.smk

@@ -1,11 +1,28 @@
 # Plan tiles (checkpoint) and segment each tile on a GPU.
 
+
+rule fetch_model:
+    """Cache the segmentation model on the (networked) submit host.
+
+    Declared local (see ``localrules`` in the Snakefile) so it never runs on an
+    offline GPU node — Cellpose downloads its weights here, into shared $HOME.
+    """
+    output:
+        touch(MODEL_OK),
+    log:
+        f"{LOGS}/fetch_model.log",
+    script:
+        "../scripts/fetch_model.py"
+
+
 checkpoint prepare:
     input:
         IMAGE_OK,
     output:
         tiles=TILES,
         stage=touch(STAGE_OK),
+    log:
+        STEPLOG,
     script:
         "../scripts/prepare_tiles.py"
 
@@ -16,7 +33,10 @@ rule segment:
         tiles=TILES,
         stage=STAGE_OK,
         image=IMAGE_OK,
+        model=MODEL_OK,
     output:
         f"{WORK}/seg/{{index}}.done",
+    log:
+        f"{LOGS}/segment/{{index}}.log",
     script:
         "../scripts/segment_tile.py"

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/scripts/_pw.py

@@ -8,11 +8,62 @@ these helpers only turn the Snakemake config into the right arguments.
 from __future__ import annotations
 
 import json
+import logging
+import sys
 from pathlib import Path
 
 from patchworks import load_ome_zarr
 
 
+class _Tee:
+    """Write to several streams at once (e.g. the SLURM log and a file)."""
+
+    def __init__(self, *streams):
+        self._streams = streams
+
+    def write(self, data):
+        for stream in self._streams:
+            stream.write(data)
+
+    def flush(self):
+        for stream in self._streams:
+            stream.flush()
+
+
+def start_log(path, *, append=True):
+    """Tee stdout/stderr (and logging) into ``path``.
+
+    Captures prints, tracebacks and library logging into a file in the work
+    directory, independent of the (often empty) SLURM job log. Line-buffered,
+    so output up to a crash or OOM kill is preserved.
+
+    Parameters
+    ----------
+    path : str or Path
+        Log file to write. Parent directories are created.
+    append : bool, optional
+        Append to an existing log (keep retry history) instead of truncating.
+        Default True.
+
+    Returns
+    -------
+    TextIO
+        The open log file (kept open for the lifetime of the process).
+    """
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    handle = open(path, "a" if append else "w", buffering=1)
+    sys.stdout = _Tee(sys.__stdout__, handle)
+    sys.stderr = _Tee(sys.__stderr__, handle)
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+        handlers=[logging.StreamHandler(sys.stdout)],
+        force=True,
+    )
+    return handle
+
+
 def open_image(work_dir, channel, level):
     """Open the converted image for segmentation.
 

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/scripts/convert.py

@@ -8,6 +8,9 @@ conversion is not redone. To force a fresh conversion, delete ``image.zarr``
 
 from patchworks.plugins.ome_zarr import to_ome_zarr
 
+from _pw import start_log
+
+start_log(snakemake.log[0])  # noqa: F821
 cfg = snakemake.config  # noqa: F821  (injected by Snakemake)
 chunks = cfg.get("convert_chunks")
 to_ome_zarr(

patchworks-0.11.7/workflow/scripts/fetch_model.py

@@ -0,0 +1,21 @@
+"""Snakemake (local) script: cache the segmentation model before segmenting.
+
+Runs on the submit host, which has network access, so the offline GPU nodes
+never try to download Cellpose weights at run time (they read the shared
+``$HOME/.cellpose`` cache instead).
+"""
+
+from _pw import start_log
+
+start_log(snakemake.log[0])  # noqa: F821
+cfg = snakemake.config  # noqa: F821
+
+if cfg.get("method", "cellpose") == "cellpose":
+    # _get_model downloads + caches the weights keyed by (model, gpu).
+    from patchworks.plugins.cellpose import _get_model
+
+    model = cfg["cellpose"]["model"]
+    _get_model({"model": model, "gpu": False})
+    print(f"[patchworks] cached segmentation model: {model}")
+else:
+    print(f"[patchworks] method={cfg.get('method')!r}; no model to prefetch")

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/scripts/merge.py

@@ -11,8 +11,9 @@ from pathlib import Path
 from patchworks import merge_tile_labels
 from patchworks.plugins.ome_zarr import write_labels
 
-from _pw import stage_path
+from _pw import stage_path, start_log
 
+start_log(snakemake.log[0])  # noqa: F821
 cfg = snakemake.config  # noqa: F821
 work_dir = cfg["work_dir"]
 image_store = str(Path(work_dir) / "image.zarr")

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/scripts/prepare_tiles.py

@@ -11,8 +11,9 @@ from patchworks import (
     spatial_tiles,
 )
 
-from _pw import open_image, stage_path
+from _pw import open_image, stage_path, start_log
 
+start_log(snakemake.log[0])  # noqa: F821
 cfg = snakemake.config  # noqa: F821
 work_dir = cfg["work_dir"]
 image = open_image(work_dir, cfg["channel"], cfg["level"])

{patchworks-0.11.5 → patchworks-0.11.7}/workflow/scripts/segment_tile.py

@@ -6,8 +6,9 @@ run in parallel. Each job writes a disjoint chunk of the stage store.
 
 from patchworks import stage_tile
 
-from _pw import build_fn, load_tiles_json, open_image, stage_path
+from _pw import build_fn, load_tiles_json, open_image, stage_path, start_log
 
+start_log(snakemake.log[0])  # noqa: F821
 cfg = snakemake.config  # noqa: F821
 index = int(snakemake.wildcards.index)  # noqa: F821
 work_dir = cfg["work_dir"]