snakemake-executor-plugin-slurm 1.1.0__tar.gz → 1.2.1__tar.gz

{snakemake_executor_plugin_slurm-1.1.0 → snakemake_executor_plugin_slurm-1.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 1.1.0
+Version: 1.2.1
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 License: MIT
 Keywords: snakemake,plugin,executor,cluster,slurm

{snakemake_executor_plugin_slurm-1.1.0 → snakemake_executor_plugin_slurm-1.2.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "snakemake-executor-plugin-slurm"
-version = "1.1.0"
+version = "1.2.1"
 description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
 authors = [
     "Christian Meesters <meesters@uni-mainz.de>",
@@ -24,8 +24,8 @@ throttler = "^1.2.2"
 black = "^23.7.0"
 flake8 = "^6.1.0"
 coverage = "^7.3.1"
-pytest = "^7.4.2"
-snakemake = "^8.20.0"
+pytest = "^8.3.5"
+snakemake = "^9.1.1"
 
 [tool.coverage.run]
 omit = [".*", "*/site-packages/*", "Snakefile"]

{snakemake_executor_plugin_slurm-1.1.0 → snakemake_executor_plugin_slurm-1.2.1}/snakemake_executor_plugin_slurm/__init__.py

@@ -26,9 +26,9 @@ from snakemake_interface_executor_plugins.jobs import (
     JobExecutorInterface,
 )
 from snakemake_interface_common.exceptions import WorkflowError
-from snakemake_executor_plugin_slurm_jobstep import get_cpu_setting
 
 from .utils import delete_slurm_environment, delete_empty_dirs, set_gres_string
+from .submit_string import get_submit_command
 
 
 @dataclass
@@ -135,9 +135,10 @@ common_settings = CommonSettings(
 # Required:
 # Implementation of your executor
 class Executor(RemoteExecutor):
-    def __post_init__(self):
+    def __post_init__(self, test_mode: bool = False):
         # run check whether we are running in a SLURM job context
         self.warn_on_jobcontext()
+        self.test_mode = test_mode
         self.run_uuid = str(uuid.uuid4())
         self.logger.info(f"SLURM run ID: {self.run_uuid}")
         self._fallback_account_arg = None
@@ -225,31 +226,28 @@ class Executor(RemoteExecutor):
             comment_str = f"rule_{job.name}"
         else:
             comment_str = f"rule_{job.name}_wildcards_{wildcard_str}"
-        call = (
-            f"sbatch "
-            f"--parsable "
-            f"--job-name {self.run_uuid} "
-            f"--output '{slurm_logfile}' "
-            f"--export=ALL "
-            f"--comment '{comment_str}'"
-        )
+        # check whether the 'slurm_extra' parameter is used correctly
+        # prior to putatively setting in the sbatch call
+        if job.resources.get("slurm_extra"):
+            self.check_slurm_extra(job)
 
-        if not self.workflow.executor_settings.no_account:
-            call += self.get_account_arg(job)
+        job_params = {
+            "run_uuid": self.run_uuid,
+            "slurm_logfile": slurm_logfile,
+            "comment_str": comment_str,
+            "account": self.get_account_arg(job),
+            "partition": self.get_partition_arg(job),
+            "workdir": self.workflow.workdir_init,
+        }
 
-        call += self.get_partition_arg(job)
+        call = get_submit_command(job, job_params)
 
         if self.workflow.executor_settings.requeue:
             call += " --requeue"
 
         call += set_gres_string(job)
 
-        if job.resources.get("clusters"):
-            call += f" --clusters {job.resources.clusters}"
-
-        if job.resources.get("runtime"):
-            call += f" -t {job.resources.runtime}"
-        else:
+        if not job.resources.get("runtime"):
             self.logger.warning(
                 "No wall time information given. This might or might not "
                 "work on your cluster. "
@@ -257,28 +255,19 @@ class Executor(RemoteExecutor):
                 "default via --default-resources."
             )
 
-        if job.resources.get("constraint"):
-            call += f" -C '{job.resources.constraint}'"
-        if job.resources.get("mem_mb_per_cpu"):
-            call += f" --mem-per-cpu {job.resources.mem_mb_per_cpu}"
-        elif job.resources.get("mem_mb"):
-            call += f" --mem {job.resources.mem_mb}"
-        else:
+        if not job.resources.get("mem_mb_per_cpu") and not job.resources.get("mem_mb"):
             self.logger.warning(
                 "No job memory information ('mem_mb' or 'mem_mb_per_cpu') is given "
                 "- submitting without. This might or might not work on your cluster."
             )
 
-        if job.resources.get("nodes", False):
-            call += f" --nodes={job.resources.get('nodes', 1)}"
-
-        # fixes #40 - set ntasks regardless of mpi, because
-        # SLURM v22.05 will require it for all jobs
-        gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
-        if gpu_job:
-            call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
-        else:
-            call += f" --ntasks={job.resources.get('tasks', 1)}"
+            # fixes #40 - set ntasks regardless of mpi, because
+            # SLURM v22.05 introduced the requirement for all jobs
+            gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
+            if gpu_job:
+                call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
+            else:
+                call += f" --ntasks={job.resources.get('tasks', 1)}"
         # MPI job
         if job.resources.get("mpi", False):
             if not job.resources.get("tasks_per_node") and not job.resources.get(
@@ -290,19 +279,8 @@ class Executor(RemoteExecutor):
                     "Probably not what you want."
                 )
 
-        # we need to set cpus-per-task OR cpus-per-gpu, the function
-        # will return a string with the corresponding value
-        call += f" {get_cpu_setting(job, gpu_job)}"
-        if job.resources.get("slurm_extra"):
-            self.check_slurm_extra(job)
-            call += f" {job.resources.slurm_extra}"
-
         exec_job = self.format_job_exec(job)
 
-        # ensure that workdir is set correctly
-        # use short argument as this is the same in all slurm versions
-        # (see https://github.com/snakemake/snakemake/issues/2014)
-        call += f" -D {self.workflow.workdir_init}"
         # and finally the job to execute with all the snakemake parameters
         call += f' --wrap="{exec_job}"'
 
@@ -656,12 +634,14 @@ We leave it to SLURM to resume your job(s)"""
         tries to deduce the acccount from recent jobs,
         returns None, if none is found
         """
-        cmd = f'sacct -nu "{os.environ["USER"]}" -o Account%256 | head -n1'
+        cmd = f'sacct -nu "{os.environ["USER"]}" -o Account%256 | tail -1'
         try:
             sacct_out = subprocess.check_output(
                 cmd, shell=True, text=True, stderr=subprocess.PIPE
             )
-            return sacct_out.replace("(null)", "").strip()
+            possible_account = sacct_out.replace("(null)", "").strip()
+            if possible_account == "none":  # some clusters may not use an account
+                return None
         except subprocess.CalledProcessError as e:
             self.logger.warning(
                 f"No account was given, not able to get a SLURM account via sacct: "
@@ -712,7 +692,7 @@ We leave it to SLURM to resume your job(s)"""
             )
             return ""
 
-        if account not in accounts:
+        if account.lower() not in accounts:
             raise WorkflowError(
                 f"The given account {account} appears to be invalid. Available "
                 f"accounts:\n{', '.join(accounts)}"

snakemake_executor_plugin_slurm-1.2.1/snakemake_executor_plugin_slurm/submit_string.py

@@ -0,0 +1,72 @@
+from snakemake_executor_plugin_slurm_jobstep import get_cpu_setting
+from types import SimpleNamespace
+
+
+def get_submit_command(job, params):
+    """
+    Return the submit command for the job.
+    """
+    # Convert params dict to a SimpleNamespace for attribute-style access
+    params = SimpleNamespace(**params)
+
+    call = (
+        f"sbatch "
+        f"--parsable "
+        f"--job-name {params.run_uuid} "
+        f'--output "{params.slurm_logfile}" '
+        f"--export=ALL "
+        f'--comment "{params.comment_str}"'
+    )
+
+    # No accout or partition checking is required, here.
+    # Checking is done in the submit function.
+
+    # here, only the string is used, as it already contains
+    # '-A {account_name}'
+    call += f" {params.account}"
+    # here, only the string is used, as it already contains
+    # '- p {partition_name}'
+    call += f" {params.partition}"
+
+    if job.resources.get("clusters"):
+        call += f" --clusters {job.resources.clusters}"
+
+    if job.resources.get("runtime"):
+        call += f" -t {job.resources.runtime}"
+
+    if job.resources.get("constraint") or isinstance(
+        job.resources.get("constraint"), str
+    ):
+        call += f" -C '{job.resources.get('constraint')}'"
+
+    if job.resources.get("qos") or isinstance(job.resources.get("qos"), str):
+        call += f" --qos='{job.resources.qos}'"
+
+    if job.resources.get("mem_mb_per_cpu"):
+        call += f" --mem-per-cpu {job.resources.mem_mb_per_cpu}"
+    elif job.resources.get("mem_mb"):
+        call += f" --mem {job.resources.mem_mb}"
+
+    if job.resources.get("nodes", False):
+        call += f" --nodes={job.resources.get('nodes', 1)}"
+
+    # fixes #40 - set ntasks regardless of mpi, because
+    # SLURM v22.05 will require it for all jobs
+    gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
+    if gpu_job:
+        call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
+    else:
+        call += f" --ntasks={job.resources.get('tasks', 1)}"
+
+    # we need to set cpus-per-task OR cpus-per-gpu, the function
+    # will return a string with the corresponding value
+    call += f" {get_cpu_setting(job, gpu_job)}"
+    if job.resources.get("slurm_extra"):
+        call += f" {job.resources.slurm_extra}"
+
+    # ensure that workdir is set correctly
+    # use short argument as this is the same in all slurm versions
+    # (see https://github.com/snakemake/snakemake/issues/2014)
+    call += f" -D '{params.workdir}'"
+
+    return call