snakemake-executor-plugin-slurm 0.9.0__tar.gz → 0.10.1__tar.gz

{snakemake_executor_plugin_slurm-0.9.0 → snakemake_executor_plugin_slurm-0.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: snakemake-executor-plugin-slurm
-Version: 0.9.0
+Version: 0.10.1
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 Home-page: https://github.com/snakemake/snakemake-executor-plugin-slurm
 License: MIT

{snakemake_executor_plugin_slurm-0.9.0 → snakemake_executor_plugin_slurm-0.10.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "snakemake-executor-plugin-slurm"
-version = "0.9.0"
+version = "0.10.1"
 description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
 authors = [
     "Christian Meesters <meesters@uni-mainz.de>",
@@ -25,7 +25,7 @@ black = "^23.7.0"
 flake8 = "^6.1.0"
 coverage = "^7.3.1"
 pytest = "^7.4.2"
-snakemake = {git = "https://github.com/snakemake/snakemake.git"}
+snakemake = "^8.20.0"
 
 [tool.coverage.run]
 omit = [".*", "*/site-packages/*", "Snakefile"]

{snakemake_executor_plugin_slurm-0.9.0 → snakemake_executor_plugin_slurm-0.10.1}/snakemake_executor_plugin_slurm/__init__.py

@@ -26,6 +26,8 @@ from snakemake_interface_executor_plugins.jobs import (
 from snakemake_interface_common.exceptions import WorkflowError
 from snakemake_executor_plugin_slurm_jobstep import get_cpus_per_task
 
+from .utils import delete_slurm_environment
+
 
 @dataclass
 class ExecutorSettings(ExecutorSettingsBase):
@@ -76,6 +78,7 @@ class Executor(RemoteExecutor):
         self.logger.info(f"SLURM run ID: {self.run_uuid}")
         self._fallback_account_arg = None
         self._fallback_partition = None
+        self._preemption_warning = False  # no preemption warning has been issued
         # providing a short-hand, even if subsequent calls seem redundant
         self.settings: ExecutorSettings = self.workflow.executor_settings
 
@@ -84,10 +87,11 @@ class Executor(RemoteExecutor):
             if "SLURM_JOB_ID" in os.environ:
                 self.logger.warning(
                     "You are running snakemake in a SLURM job context. "
-                    "This is not recommended, as it may lead to unexpected behavior."
+                    "This is not recommended, as it may lead to unexpected behavior. "
                     "Please run Snakemake directly on the login node."
                 )
                 time.sleep(5)
+                delete_slurm_environment()
         done = True
 
     def additional_general_args(self):
@@ -130,7 +134,11 @@ class Executor(RemoteExecutor):
         else:
             comment_str = f"rule_{job.name}_wildcards_{wildcard_str}"
         call = (
-            f"sbatch --job-name {self.run_uuid} --output {slurm_logfile} --export=ALL "
+            f"sbatch "
+            f"--parsable "
+            f"--job-name {self.run_uuid} "
+            f"--output '{slurm_logfile}' "
+            f"--export=ALL "
             f"--comment {comment_str}"
         )
 
@@ -205,10 +213,11 @@ class Executor(RemoteExecutor):
             )
 
         # multicluster submissions yield submission infos like
-        # "Submitted batch job <id> on cluster <name>".
-        # To extract the job id in this case we need to match any number
-        # in between a string - which might change in future versions of SLURM.
-        slurm_jobid = re.search(r"\d+", out).group()
+        # "Submitted batch job <id> on cluster <name>" by default, but with the
+        # --parsable option it simply yields "<id>;<name>".
+        # To extract the job id we split by semicolon and take the first element
+        # (this also works if no cluster name was provided)
+        slurm_jobid = out.split(";")[0]
         slurm_logfile = slurm_logfile.replace("%j", slurm_jobid)
         self.logger.info(
             f"Job {job.jobid} has been submitted with SLURM jobid {slurm_jobid} "
@@ -243,7 +252,6 @@ class Executor(RemoteExecutor):
             "FAILED",
             "NODE_FAIL",
             "OUT_OF_MEMORY",
-            "PREEMPTED",
             "TIMEOUT",
             "ERROR",
         )
@@ -343,6 +351,16 @@ class Executor(RemoteExecutor):
                     self.report_job_success(j)
                     any_finished = True
                     active_jobs_seen_by_sacct.remove(j.external_jobid)
+                elif status == "PREEMPTED" and not self._preemption_warning:
+                    self._preemption_warning = True
+                    self.logger.warning(
+                        """
+===== A Job preemption  occured! =====
+Leave Snakemake running, if possible. Otherwise Snakemake
+needs to restart this job upon a Snakemake restart.
+
+We leave it to SLURM to resume your job(s)"""
+                    )
                 elif status == "UNKNOWN":
                     # the job probably does not exist anymore, but 'sacct' did not work
                     # so we assume it is finished
@@ -390,6 +408,14 @@ class Executor(RemoteExecutor):
                 )
             except subprocess.TimeoutExpired:
                 self.logger.warning("Unable to cancel jobs within a minute.")
+            except subprocess.CalledProcessError as e:
+                msg = e.stderr.strip()
+                if msg:
+                    msg = f": {msg}"
+                raise WorkflowError(
+                    "Unable to cancel jobs with scancel "
+                    f"(exit code {e.returncode}){msg}"
+                ) from e
 
     async def job_stati(self, command):
         """Obtain SLURM job status of all submitted jobs with sacct

snakemake_executor_plugin_slurm-0.10.1/snakemake_executor_plugin_slurm/utils.py

@@ -0,0 +1,16 @@
+# utility functions for the SLURM executor plugin
+
+import os
+
+
+def delete_slurm_environment():
+    """
+    Function to delete all environment variables
+    starting with 'SLURM_'. The parent shell will
+    still have this environment. This is needed to
+    submit within a SLURM job context to avoid
+    conflicting environments.
+    """
+    for var in os.environ:
+        if var.startswith("SLURM_"):
+            del os.environ[var]