snakemake-executor-plugin-slurm 1.4.0__tar.gz → 1.6.0__tar.gz

{snakemake_executor_plugin_slurm-1.4.0 → snakemake_executor_plugin_slurm-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 1.4.0
+Version: 1.6.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 License: MIT
 Keywords: snakemake,plugin,executor,cluster,slurm
@@ -12,6 +12,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: numpy (>=1.26.4,<3)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: snakemake-executor-plugin-slurm-jobstep (>=0.3.0,<0.4.0)
 Requires-Dist: snakemake-interface-common (>=1.13.0,<2.0.0)
 Requires-Dist: snakemake-interface-executor-plugins (>=9.1.1,<10.0.0)

{snakemake_executor_plugin_slurm-1.4.0 → snakemake_executor_plugin_slurm-1.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "snakemake-executor-plugin-slurm"
-version = "1.4.0"
+version = "1.6.0"
 description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
 authors = [
     "Christian Meesters <meesters@uni-mainz.de>",
@@ -18,6 +18,8 @@ python = "^3.11"
 snakemake-interface-common = "^1.13.0"
 snakemake-interface-executor-plugins = "^9.1.1"
 snakemake-executor-plugin-slurm-jobstep = "^0.3.0"
+pandas = "^2.2.3"
+numpy = ">=1.26.4, <3"
 throttler = "^1.2.2"
 
 [tool.poetry.group.dev.dependencies]
@@ -25,7 +27,8 @@ black = "^23.7.0"
 flake8 = "^6.1.0"
 coverage = "^7.3.1"
 pytest = "^8.3.5"
-snakemake = "^9.4.0"
+snakemake = "^9.6.0"
+pandas = "^2.2.3"
 
 [tool.coverage.run]
 omit = [".*", "*/site-packages/*", "Snakefile"]

{snakemake_executor_plugin_slurm-1.4.0 → snakemake_executor_plugin_slurm-1.6.0}/snakemake_executor_plugin_slurm/__init__.py

@@ -3,7 +3,6 @@ __copyright__ = "Copyright 2023, David Lähnemann, Johannes Köster, Christian M
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 
-import atexit
 import csv
 from io import StringIO
 import os
@@ -16,6 +15,7 @@ from dataclasses import dataclass, field
 from datetime import datetime, timedelta
 from typing import List, Generator, Optional
 import uuid
+
 from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo
 from snakemake_interface_executor_plugins.executors.remote import RemoteExecutor
 from snakemake_interface_executor_plugins.settings import (
@@ -27,7 +27,12 @@ from snakemake_interface_executor_plugins.jobs import (
 )
 from snakemake_interface_common.exceptions import WorkflowError
 
-from .utils import delete_slurm_environment, delete_empty_dirs, set_gres_string
+from .utils import (
+    delete_slurm_environment,
+    delete_empty_dirs,
+    set_gres_string,
+)
+from .efficiency_report import create_efficiency_report
 from .submit_string import get_submit_command
 
 
@@ -106,6 +111,35 @@ class ExecutorSettings(ExecutorSettingsBase):
             "required": False,
         },
     )
+    efficiency_report: bool = field(
+        default=False,
+        metadata={
+            "help": "Generate an efficiency report at the end of the workflow. "
+            "This flag has no effect, if not set.",
+            "env_var": False,
+            "required": False,
+        },
+    )
+    efficiency_report_path: Optional[Path] = field(
+        default=None,
+        metadata={
+            "help": "Path to the efficiency report file. "
+            "If not set, the report will be written to "
+            "the current working directory with the name "
+            "'efficiency_report_<run_uuid>.csv'. "
+            "This flag has no effect, if not set.",
+            "env_var": False,
+            "required": False,
+        },
+    )
+    efficiency_threshold: Optional[float] = field(
+        default=0.8,
+        metadata={
+            "help": "The efficiency threshold for the efficiency report. "
+            "Jobs with an efficiency below this threshold will be reported. "
+            "This flag has no effect, if not set.",
+        },
+    )
     reservation: Optional[str] = field(
         default=None,
         metadata={
@@ -157,7 +191,26 @@ class Executor(RemoteExecutor):
             if self.workflow.executor_settings.logdir
             else Path(".snakemake/slurm_logs").resolve()
         )
-        atexit.register(self.clean_old_logs)
+
+    def shutdown(self) -> None:
+        """
+        Shutdown the executor.
+        This method is overloaded, to include the cleaning of old log files
+        and to optionally create an efficiency report.
+        """
+        # First, we invoke the original shutdown method
+        super().shutdown()
+
+        # Next, clean up old log files, unconditionally.
+        self.clean_old_logs()
+        # If the efficiency report is enabled, create it.
+        if self.workflow.executor_settings.efficiency_report:
+            create_efficiency_report(
+                e_threshold=self.workflow.executor_settings.efficiency_threshold,
+                run_uuid=self.run_uuid,
+                e_report_path=self.workflow.executor_settings.efficiency_report_path,
+                logger=self.logger,
+            )
 
     def clean_old_logs(self) -> None:
         """Delete files older than specified age from the SLURM log directory."""
@@ -168,7 +221,8 @@ class Executor(RemoteExecutor):
             return
         cutoff_secs = age_cutoff * 86400
         current_time = time.time()
-        self.logger.info(f"Cleaning up log files older than {age_cutoff} day(s)")
+        self.logger.info(f"Cleaning up log files older than {age_cutoff} day(s).")
+
         for path in self.slurm_logdir.rglob("*.log"):
             if path.is_file():
                 try:
@@ -176,12 +230,14 @@ class Executor(RemoteExecutor):
                     if file_age > cutoff_secs:
                         path.unlink()
                 except (OSError, FileNotFoundError) as e:
-                    self.logger.warning(f"Could not delete logfile {path}: {e}")
+                    self.logger.error(f"Could not delete logfile {path}: {e}")
         # we need a 2nd iteration to remove putatively empty directories
         try:
             delete_empty_dirs(self.slurm_logdir)
         except (OSError, FileNotFoundError) as e:
-            self.logger.warning(f"Could not delete empty directory {path}: {e}")
+            self.logger.error(
+                f"Could not delete empty directories in {self.slurm_logdir}: {e}"
+            )
 
     def warn_on_jobcontext(self, done=None):
         if not done:
@@ -310,9 +366,15 @@ class Executor(RemoteExecutor):
                     process.returncode, call, output=err
                 )
         except subprocess.CalledProcessError as e:
-            raise WorkflowError(
-                f"SLURM sbatch failed. The error message was {e.output}"
+            self.report_job_error(
+                SubmittedJobInfo(job),
+                msg=(
+                    "SLURM sbatch failed. "
+                    f"The error message was '{e.output.strip()}'.\n"
+                    f"    sbatch call:\n        {call}\n"
+                ),
             )
+            return
         # any other error message indicating failure?
         if "submission failed" in err:
             raise WorkflowError(
@@ -389,7 +451,7 @@ class Executor(RemoteExecutor):
 
         # We use this sacct syntax for argument 'starttime' to keep it compatible
         # with slurm < 20.11
-        sacct_starttime = f"{datetime.now() - timedelta(days = 2):%Y-%m-%dT%H:00}"
+        sacct_starttime = f"{datetime.now() - timedelta(days=2):%Y-%m-%dT%H:00}"
         # previously we had
         # f"--starttime now-2days --endtime now --name {self.run_uuid}"
         # in line 218 - once v20.11 is definitively not in use any more,
@@ -741,10 +803,10 @@ We leave it to SLURM to resume your job(s)"""
         jobname = re.compile(r"--job-name[=?|\s+]|-J\s?")
         if re.search(jobname, job.resources.slurm_extra):
             raise WorkflowError(
-                "The --job-name option is not allowed in the 'slurm_extra' "
-                "parameter. The job name is set by snakemake and must not be "
-                "overwritten. It is internally used to check the stati of the "
-                "all submitted jobs by this workflow."
+                "The --job-name option is not allowed in the 'slurm_extra' parameter. "
+                "The job name is set by snakemake and must not be overwritten. "
+                "It is internally used to check the stati of the all submitted jobs "
+                "by this workflow."
                 "Please consult the documentation if you are unsure how to "
                 "query the status of your jobs."
             )

snakemake_executor_plugin_slurm-1.6.0/snakemake_executor_plugin_slurm/efficiency_report.py

@@ -0,0 +1,185 @@
+import re
+import pandas as pd
+from pathlib import Path
+import subprocess
+import shlex
+
+import os  # only temporarily needed for printf debugging
+import numpy as np
+
+
+def time_to_seconds(time_str):
+    """Convert SLURM time format to seconds."""
+    if pd.isna(time_str) or time_str.strip() == "":
+        return 0
+    parts = time_str.split(":")
+
+    if len(parts) == 3:  # H:M:S
+        return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
+    elif len(parts) == 2:  # M:S
+        return int(parts[0]) * 60 + float(parts[1])
+    elif len(parts) == 1:  # S
+        return float(parts[0])
+    return 0
+
+
+def parse_maxrss(maxrss):
+    """Convert MaxRSS to MB."""
+    if pd.isna(maxrss) or maxrss.strip() == "" or maxrss == "0":
+        return 0
+    match = re.match(r"(\d+(?:\.\d+)?)([KMG]?)", maxrss)
+    if match:
+        value, unit = match.groups()
+        value = float(value)
+        unit_multipliers = {"K": 1 / 1024, "M": 1, "G": 1024}
+        return value * unit_multipliers.get(unit, 1)
+    return 0
+
+
+def parse_reqmem(reqmem, number_of_nodes=1):
+    """Convert requested memory to MB."""
+    if pd.isna(reqmem) or reqmem.strip() == "":
+        return 0
+    # 4Gc (per-CPU) / 16Gn (per-node) / 2.5G
+    match = re.match(r"(\d+(?:\.\d+)?)([KMG])?([cn]|/node)?", reqmem)
+    if match:
+        value, unit, per_unit = match.groups()
+        value = float(value)
+        unit_multipliers = {"K": 1 / 1024, "M": 1, "G": 1024}
+        mem_mb = value * unit_multipliers.get(unit, 1)
+        if per_unit in ("n", "/node"):  # per-node
+            nodes = 1 if pd.isna(number_of_nodes) else number_of_nodes
+            return mem_mb * nodes
+        # `/c` or `c` → per-CPU; caller may multiply later
+        return mem_mb  # Default case (per CPU or total)
+    return 0
+
+
+def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
+    """
+    Fetch sacct job data for a Snakemake workflow
+    and compute efficiency metrics.
+    """
+    cmd = f"sacct --name={run_uuid} --parsable2 --noheader"
+    cmd += (
+        " --format=JobID,JobName,Comment,Elapsed,TotalCPU," "NNodes,NCPUS,MaxRSS,ReqMem"
+    )
+
+    try:
+        result = subprocess.run(
+            shlex.split(cmd), capture_output=True, text=True, check=True
+        )
+        raw = result.stdout.strip()
+        if not raw:
+            logger.warning(f"No job data found for workflow {run_uuid}.")
+            return None
+        lines = raw.split("\n")
+
+    except subprocess.CalledProcessError:
+        logger.error(f"Failed to retrieve job data for workflow {run_uuid}.")
+        return None
+
+    # Convert to DataFrame
+    df = pd.DataFrame(
+        (line.split("|") for line in lines),
+        columns=[
+            "JobID",
+            "JobName",
+            "Comment",
+            "Elapsed",
+            "TotalCPU",
+            "NNodes",
+            "NCPUS",
+            "MaxRSS",
+            "ReqMem",
+        ],
+    )
+
+    # If the "Comment" column is empty,
+    # a) delete the column
+    # b) issue a warning
+    if df["Comment"].replace("", pd.NA).isna().all():
+        logger.warning(
+            f"No comments found for workflow {run_uuid}. "
+            "This field is used to store the rule name. "
+            "Please ensure that the 'comment' field is set for your cluster. "
+            "Administrators can set this up in the SLURM configuration."
+        )
+        df.drop(columns=["Comment"], inplace=True)
+        # remember, that the comment column is not available
+        nocomment = True
+    # else: rename the column to 'RuleName'
+    else:
+        df.rename(columns={"Comment": "RuleName"}, inplace=True)
+        nocomment = False
+    # Convert types
+    df["NNodes"] = pd.to_numeric(df["NNodes"], errors="coerce")
+    df["NCPUS"] = pd.to_numeric(df["NCPUS"], errors="coerce")
+
+    # Convert time fields
+    df["Elapsed_sec"] = df["Elapsed"].apply(time_to_seconds)
+    df["TotalCPU_sec"] = df["TotalCPU"].apply(time_to_seconds)
+
+    # Compute CPU efficiency
+    df["CPU Efficiency (%)"] = (
+        df["TotalCPU_sec"]
+        / (df["Elapsed_sec"].clip(lower=1) * df["NCPUS"].clip(lower=1))
+    ) * 100
+    df.replace([np.inf, -np.inf], 0, inplace=True)
+
+    # Convert MaxRSS
+    df["MaxRSS_MB"] = df["MaxRSS"].apply(parse_maxrss)
+
+    # Convert ReqMem and calculate memory efficiency
+    df["RequestedMem_MB"] = df.apply(
+        lambda row: parse_reqmem(row["ReqMem"], row["NNodes"]), axis=1
+    )
+    df["Memory Usage (%)"] = df.apply(
+        lambda row: (
+            (row["MaxRSS_MB"] / row["RequestedMem_MB"] * 100)
+            if row["RequestedMem_MB"] > 0
+            else 0
+        ),
+        axis=1,
+    )
+
+    df["Memory Usage (%)"] = df["Memory Usage (%)"].fillna(0).round(2)
+
+    # Drop all rows containing "batch" or "extern" as job names
+    df = df[~df["JobName"].str.contains("batch|extern", na=False)]
+
+    # Log warnings for low efficiency
+    for _, row in df.iterrows():
+        if row["CPU Efficiency (%)"] < e_threshold:
+            if nocomment:
+                logger.warning(
+                    f"Job {row['JobID']} ({row['JobName']}) "
+                    f"has low CPU efficiency: {row['CPU Efficiency (%)']}%."
+                )
+            else:
+                # if the comment column is available, we can use it to
+                # identify the rule name
+                logger.warning(
+                    f"Job {row['JobID']} for rule '{row['RuleName']}' "
+                    f"({row['JobName']}) has low CPU efficiency: "
+                    f"{row['CPU Efficiency (%)']}%."
+                )
+
+    # we construct a path object to allow for a customi
+    # logdir, if specified
+    p = Path()
+
+    # Save the report to a CSV file
+    logfile = f"efficiency_report_{run_uuid}.csv"
+    if e_report_path:
+        logfile = Path(e_report_path) / logfile
+    else:
+        logfile = p.cwd() / logfile
+    # ensure the directory exists
+    logfile.parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(logfile)
+
+    # write out the efficiency report at normal verbosity in any case
+    logger.info(f"Efficiency report for workflow {run_uuid} saved to {logfile}.")
+    # state directory contents for debugging purposes
+    logger.debug(f"Current directory contents in '{p.cwd()}': {os.listdir(p.cwd())}")

{snakemake_executor_plugin_slurm-1.4.0 → snakemake_executor_plugin_slurm-1.6.0}/snakemake_executor_plugin_slurm/submit_string.py

@@ -50,12 +50,19 @@ def get_submit_command(job, params):
     if job.resources.get("nodes", False):
         call += f" --nodes={job.resources.get('nodes', 1)}"
 
-    # fixes #40 - set ntasks regardless of mpi, because
-    # SLURM v22.05 will require it for all jobs
     gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
     if gpu_job:
-        call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
+        # fixes #316 - allow unsetting of tasks per gpu
+        # apparently, python's internal process manangement interfers with SLURM
+        # e.g. for pytorch
+        ntasks_per_gpu = job.resources.get(
+            "tasks_per_gpu", job.resources.get("tasks", 1)
+        )
+        if ntasks_per_gpu >= 1:
+            call += f" --ntasks-per-gpu={ntasks_per_gpu}"
     else:
+        # fixes #40 - set ntasks regardless of mpi, because
+        # SLURM v22.05 will require it for all jobs
         call += f" --ntasks={job.resources.get('tasks', 1)}"
 
     # we need to set cpus-per-task OR cpus-per-gpu, the function