snakemake-executor-plugin-slurm 0.7.0__tar.gz → 0.9.0__tar.gz

{snakemake_executor_plugin_slurm-0.7.0 → snakemake_executor_plugin_slurm-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: snakemake-executor-plugin-slurm
-Version: 0.7.0
+Version: 0.9.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 Home-page: https://github.com/snakemake/snakemake-executor-plugin-slurm
 License: MIT

{snakemake_executor_plugin_slurm-0.7.0 → snakemake_executor_plugin_slurm-0.9.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "snakemake-executor-plugin-slurm"
-version = "0.7.0"
+version = "0.9.0"
 description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
 authors = [
     "Christian Meesters <meesters@uni-mainz.de>",

{snakemake_executor_plugin_slurm-0.7.0 → snakemake_executor_plugin_slurm-0.9.0}/snakemake_executor_plugin_slurm/__init__.py

@@ -7,14 +7,19 @@ import csv
 from io import StringIO
 import os
 import re
+import shlex
 import subprocess
 import time
+from dataclasses import dataclass, field
 from datetime import datetime, timedelta
-from typing import List, Generator
+from typing import List, Generator, Optional
 import uuid
 from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo
 from snakemake_interface_executor_plugins.executors.remote import RemoteExecutor
-from snakemake_interface_executor_plugins.settings import CommonSettings
+from snakemake_interface_executor_plugins.settings import (
+    ExecutorSettingsBase,
+    CommonSettings,
+)
 from snakemake_interface_executor_plugins.jobs import (
     JobExecutorInterface,
 )
@@ -22,6 +27,21 @@ from snakemake_interface_common.exceptions import WorkflowError
 from snakemake_executor_plugin_slurm_jobstep import get_cpus_per_task
 
 
+@dataclass
+class ExecutorSettings(ExecutorSettingsBase):
+    init_seconds_before_status_checks: Optional[int] = field(
+        default=40,
+        metadata={
+            "help": """
+                    Defines the time in seconds before the first status
+                    check is performed after job submission.
+                    """,
+            "env_var": False,
+            "required": False,
+        },
+    )
+
+
 # Required:
 # Specify common settings shared by various executors.
 common_settings = CommonSettings(
@@ -56,14 +76,16 @@ class Executor(RemoteExecutor):
         self.logger.info(f"SLURM run ID: {self.run_uuid}")
         self._fallback_account_arg = None
         self._fallback_partition = None
+        # providing a short-hand, even if subsequent calls seem redundant
+        self.settings: ExecutorSettings = self.workflow.executor_settings
 
     def warn_on_jobcontext(self, done=None):
         if not done:
             if "SLURM_JOB_ID" in os.environ:
                 self.logger.warning(
-                    "Running Snakemake in a SLURM within a job may lead"
-                    " to unexpected behavior. Please run Snakemake directly"
-                    " on the head node."
+                    "You are running snakemake in a SLURM job context. "
+                    "This is not recommended, as it may lead to unexpected behavior."
+                    "Please run Snakemake directly on the login node."
                 )
                 time.sleep(5)
         done = True
@@ -115,6 +137,9 @@ class Executor(RemoteExecutor):
         call += self.get_account_arg(job)
         call += self.get_partition_arg(job)
 
+        if job.resources.get("clusters"):
+            call += f" --clusters {job.resources.clusters}"
+
         if job.resources.get("runtime"):
             call += f" -t {job.resources.runtime}"
         else:
@@ -126,7 +151,7 @@ class Executor(RemoteExecutor):
             )
 
         if job.resources.get("constraint"):
-            call += f" -C {job.resources.constraint}"
+            call += f" -C '{job.resources.constraint}'"
         if job.resources.get("mem_mb_per_cpu"):
             call += f" --mem-per-cpu {job.resources.mem_mb_per_cpu}"
         elif job.resources.get("mem_mb"):
@@ -140,7 +165,7 @@ class Executor(RemoteExecutor):
         if job.resources.get("nodes", False):
             call += f" --nodes={job.resources.get('nodes', 1)}"
 
-        # fixes #40 - set ntasks regarlless of mpi, because
+        # fixes #40 - set ntasks regardless of mpi, because
         # SLURM v22.05 will require it for all jobs
         call += f" --ntasks={job.resources.get('tasks', 1)}"
         # MPI job
@@ -179,7 +204,11 @@ class Executor(RemoteExecutor):
                 f"SLURM job submission failed. The error message was {e.output}"
             )
 
-        slurm_jobid = out.split(" ")[-1]
+        # multicluster submissions yield submission infos like
+        # "Submitted batch job <id> on cluster <name>".
+        # To extract the job id in this case we need to match any number
+        # in between a string - which might change in future versions of SLURM.
+        slurm_jobid = re.search(r"\d+", out).group()
         slurm_logfile = slurm_logfile.replace("%j", slurm_jobid)
         self.logger.info(
             f"Job {job.jobid} has been submitted with SLURM jobid {slurm_jobid} "
@@ -195,7 +224,6 @@ class Executor(RemoteExecutor):
         self, active_jobs: List[SubmittedJobInfo]
     ) -> Generator[SubmittedJobInfo, None, None]:
         # Check the status of active jobs.
-
         # You have to iterate over the given list active_jobs.
         # For jobs that have finished successfully, you have to call
         # self.report_job_success(job).
@@ -244,15 +272,22 @@ class Executor(RemoteExecutor):
         # in line 218 - once v20.11 is definitively not in use any more,
         # the more readable version ought to be re-adapted
 
+        # -X: only show main job, no substeps
+        sacct_command = f"""sacct -X --parsable2 \
+                        --clusters all \
+                        --noheader --format=JobIdRaw,State \
+                        --starttime {sacct_starttime} \
+                        --endtime now --name {self.run_uuid}"""
+
+        # for better redability in verbose output
+        sacct_command = " ".join(shlex.split(sacct_command))
+
         # this code is inspired by the snakemake profile:
         # https://github.com/Snakemake-Profiles/slurm
         for i in range(status_attempts):
             async with self.status_rate_limiter:
                 (status_of_jobs, sacct_query_duration) = await self.job_stati(
-                    # -X: only show main job, no substeps
-                    f"sacct -X --parsable2 --noheader --format=JobIdRaw,State "
-                    f"--starttime {sacct_starttime} "
-                    f"--endtime now --name {self.run_uuid}"
+                    sacct_command
                 )
                 if status_of_jobs is None and sacct_query_duration is None:
                     self.logger.debug(f"could not check status of job {self.run_uuid}")
@@ -344,8 +379,10 @@ class Executor(RemoteExecutor):
                 # about 30 sec, but can be longer in extreme cases.
                 # Under 'normal' circumstances, 'scancel' is executed in
                 # virtually no time.
+                scancel_command = f"scancel {jobids} --clusters=all"
+
                 subprocess.check_output(
-                    f"scancel {jobids}",
+                    scancel_command,
                     text=True,
                     shell=True,
                     timeout=60,
@@ -509,10 +546,10 @@ class Executor(RemoteExecutor):
         jobname = re.compile(r"--job-name[=?|\s+]|-J\s?")
         if re.search(jobname, job.resources.slurm_extra):
             raise WorkflowError(
-                "The '--job-name' option is not allowed in the 'slurm_extra' "
+                "The --job-name option is not allowed in the 'slurm_extra' "
                 "parameter. The job name is set by snakemake and must not be "
-                "overwritten. It is internally used to check the stati of all "
-                "submitted jobs by this workflow."
+                "overwritten. It is internally used to check the stati of the "
+                "all submitted jobs by this workflow."
                 "Please consult the documentation if you are unsure how to "
                 "query the status of your jobs."
             )