snakemake-executor-plugin-slurm 0.8.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

snakemake_executor_plugin_slurm/__init__.py

@@ -7,6 +7,7 @@ import csv
 from io import StringIO
 import os
 import re
+import shlex
 import subprocess
 import time
 from dataclasses import dataclass, field
@@ -25,6 +26,8 @@ from snakemake_interface_executor_plugins.jobs import (
 from snakemake_interface_common.exceptions import WorkflowError
 from snakemake_executor_plugin_slurm_jobstep import get_cpus_per_task
 
+from .utils import delete_slurm_environment
+
 
 @dataclass
 class ExecutorSettings(ExecutorSettingsBase):
@@ -75,6 +78,7 @@ class Executor(RemoteExecutor):
         self.logger.info(f"SLURM run ID: {self.run_uuid}")
         self._fallback_account_arg = None
         self._fallback_partition = None
+        self._preemption_warning = False  # no preemption warning has been issued
         # providing a short-hand, even if subsequent calls seem redundant
         self.settings: ExecutorSettings = self.workflow.executor_settings
 
@@ -83,10 +87,11 @@ class Executor(RemoteExecutor):
             if "SLURM_JOB_ID" in os.environ:
                 self.logger.warning(
                     "You are running snakemake in a SLURM job context. "
-                    "This is not recommended, as it may lead to unexpected behavior."
+                    "This is not recommended, as it may lead to unexpected behavior. "
                     "Please run Snakemake directly on the login node."
                 )
                 time.sleep(5)
+                delete_slurm_environment()
         done = True
 
     def additional_general_args(self):
@@ -129,13 +134,20 @@ class Executor(RemoteExecutor):
         else:
             comment_str = f"rule_{job.name}_wildcards_{wildcard_str}"
         call = (
-            f"sbatch --job-name {self.run_uuid} --output {slurm_logfile} --export=ALL "
+            f"sbatch "
+            f"--parsable "
+            f"--job-name {self.run_uuid} "
+            f"--output {slurm_logfile} "
+            f"--export=ALL "
             f"--comment {comment_str}"
         )
 
         call += self.get_account_arg(job)
         call += self.get_partition_arg(job)
 
+        if job.resources.get("clusters"):
+            call += f" --clusters {job.resources.clusters}"
+
         if job.resources.get("runtime"):
             call += f" -t {job.resources.runtime}"
         else:
@@ -147,7 +159,7 @@ class Executor(RemoteExecutor):
             )
 
         if job.resources.get("constraint"):
-            call += f" -C {job.resources.constraint}"
+            call += f" -C '{job.resources.constraint}'"
         if job.resources.get("mem_mb_per_cpu"):
             call += f" --mem-per-cpu {job.resources.mem_mb_per_cpu}"
         elif job.resources.get("mem_mb"):
@@ -200,7 +212,12 @@ class Executor(RemoteExecutor):
                 f"SLURM job submission failed. The error message was {e.output}"
             )
 
-        slurm_jobid = out.split(" ")[-1]
+        # multicluster submissions yield submission infos like
+        # "Submitted batch job <id> on cluster <name>" by default, but with the
+        # --parsable option it simply yields "<id>;<name>".
+        # To extract the job id we split by semicolon and take the first element
+        # (this also works if no cluster name was provided)
+        slurm_jobid = out.split(";")[0]
         slurm_logfile = slurm_logfile.replace("%j", slurm_jobid)
         self.logger.info(
             f"Job {job.jobid} has been submitted with SLURM jobid {slurm_jobid} "
@@ -235,7 +252,6 @@ class Executor(RemoteExecutor):
             "FAILED",
             "NODE_FAIL",
             "OUT_OF_MEMORY",
-            "PREEMPTED",
             "TIMEOUT",
             "ERROR",
         )
@@ -264,15 +280,22 @@ class Executor(RemoteExecutor):
         # in line 218 - once v20.11 is definitively not in use any more,
         # the more readable version ought to be re-adapted
 
+        # -X: only show main job, no substeps
+        sacct_command = f"""sacct -X --parsable2 \
+                        --clusters all \
+                        --noheader --format=JobIdRaw,State \
+                        --starttime {sacct_starttime} \
+                        --endtime now --name {self.run_uuid}"""
+
+        # for better redability in verbose output
+        sacct_command = " ".join(shlex.split(sacct_command))
+
         # this code is inspired by the snakemake profile:
         # https://github.com/Snakemake-Profiles/slurm
         for i in range(status_attempts):
             async with self.status_rate_limiter:
                 (status_of_jobs, sacct_query_duration) = await self.job_stati(
-                    # -X: only show main job, no substeps
-                    f"sacct -X --parsable2 --noheader --format=JobIdRaw,State "
-                    f"--starttime {sacct_starttime} "
-                    f"--endtime now --name {self.run_uuid}"
+                    sacct_command
                 )
                 if status_of_jobs is None and sacct_query_duration is None:
                     self.logger.debug(f"could not check status of job {self.run_uuid}")
@@ -328,6 +351,16 @@ class Executor(RemoteExecutor):
                     self.report_job_success(j)
                     any_finished = True
                     active_jobs_seen_by_sacct.remove(j.external_jobid)
+                elif status == "PREEMPTED" and not self._preemption_warning:
+                    self._preemption_warning = True
+                    self.logger.warning(
+                        """
+===== A Job preemption  occured! =====
+Leave Snakemake running, if possible. Otherwise Snakemake
+needs to restart this job upon a Snakemake restart.
+
+We leave it to SLURM to resume your job(s)"""
+                    )
                 elif status == "UNKNOWN":
                     # the job probably does not exist anymore, but 'sacct' did not work
                     # so we assume it is finished
@@ -364,8 +397,10 @@ class Executor(RemoteExecutor):
                 # about 30 sec, but can be longer in extreme cases.
                 # Under 'normal' circumstances, 'scancel' is executed in
                 # virtually no time.
+                scancel_command = f"scancel {jobids} --clusters=all"
+
                 subprocess.check_output(
-                    f"scancel {jobids}",
+                    scancel_command,
                     text=True,
                     shell=True,
                     timeout=60,

snakemake_executor_plugin_slurm/utils.py

@@ -0,0 +1,16 @@
+# utility functions for the SLURM executor plugin
+
+import os
+
+
+def delete_slurm_environment():
+    """
+    Function to delete all environment variables
+    starting with 'SLURM_'. The parent shell will
+    still have this environment. This is needed to
+    submit within a SLURM job context to avoid
+    conflicting environments.
+    """
+    for var in os.environ:
+        if var.startswith("SLURM_"):
+            del os.environ[var]

{snakemake_executor_plugin_slurm-0.8.0.dist-info → snakemake_executor_plugin_slurm-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: snakemake-executor-plugin-slurm
-Version: 0.8.0
+Version: 0.10.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 Home-page: https://github.com/snakemake/snakemake-executor-plugin-slurm
 License: MIT

snakemake_executor_plugin_slurm-0.10.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+snakemake_executor_plugin_slurm/__init__.py,sha256=VMIZpkp-R61GKq2jXoLxElxfSguHwgCsv0zv8-usHQY,24229
+snakemake_executor_plugin_slurm/utils.py,sha256=DuJdFJsAmvFsrnpyb8kMoqxTEEmTsEVxroDS1t9qOGw,434
+snakemake_executor_plugin_slurm-0.10.0.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
+snakemake_executor_plugin_slurm-0.10.0.dist-info/METADATA,sha256=_88ZsjTcrcyn0m2OY1LryvmW_oJY0fzWQMeBNI8lnjk,1381
+snakemake_executor_plugin_slurm-0.10.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+snakemake_executor_plugin_slurm-0.10.0.dist-info/RECORD,,

snakemake_executor_plugin_slurm-0.8.0.dist-info/RECORD

@@ -1,5 +0,0 @@
-snakemake_executor_plugin_slurm/__init__.py,sha256=GC5yU3EsnBJBC9Z6gIQdt2GHK3QLdF0sQj5TDI6VDLo,22851
-snakemake_executor_plugin_slurm-0.8.0.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
-snakemake_executor_plugin_slurm-0.8.0.dist-info/METADATA,sha256=S2aTNWZg3rDSECsTQISccHHvxkc83YyPLugtIuHKdUk,1380
-snakemake_executor_plugin_slurm-0.8.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-snakemake_executor_plugin_slurm-0.8.0.dist-info/RECORD,,