snakemake-executor-plugin-slurm 1.0.1__py3-none-any.whl → 1.2.0__py3-none-any.whl

snakemake_executor_plugin_slurm/__init__.py

@@ -26,9 +26,9 @@ from snakemake_interface_executor_plugins.jobs import (
     JobExecutorInterface,
 )
 from snakemake_interface_common.exceptions import WorkflowError
-from snakemake_executor_plugin_slurm_jobstep import get_cpu_setting
 
 from .utils import delete_slurm_environment, delete_empty_dirs, set_gres_string
+from .submit_string import get_submit_command
 
 
 @dataclass
@@ -74,6 +74,18 @@ class ExecutorSettings(ExecutorSettingsBase):
             "required": False,
         },
     )
+    status_attempts: Optional[int] = field(
+        default=5,
+        metadata={
+            "help": "Defines the number of attempts to query the status of "
+            "all active jobs. If the status query fails, the next attempt "
+            "will be performed after the next status check interval."
+            "The default is 5 status attempts before giving up. The maximum "
+            "time between status checks is 180 seconds.",
+            "env_var": False,
+            "required": False,
+        },
+    )
     requeue: bool = field(
         default=False,
         metadata={
@@ -123,9 +135,10 @@ common_settings = CommonSettings(
 # Required:
 # Implementation of your executor
 class Executor(RemoteExecutor):
-    def __post_init__(self):
+    def __post_init__(self, test_mode: bool = False):
         # run check whether we are running in a SLURM job context
         self.warn_on_jobcontext()
+        self.test_mode = test_mode
         self.run_uuid = str(uuid.uuid4())
         self.logger.info(f"SLURM run ID: {self.run_uuid}")
         self._fallback_account_arg = None
@@ -213,31 +226,28 @@ class Executor(RemoteExecutor):
             comment_str = f"rule_{job.name}"
         else:
             comment_str = f"rule_{job.name}_wildcards_{wildcard_str}"
-        call = (
-            f"sbatch "
-            f"--parsable "
-            f"--job-name {self.run_uuid} "
-            f"--output '{slurm_logfile}' "
-            f"--export=ALL "
-            f"--comment '{comment_str}'"
-        )
+        # check whether the 'slurm_extra' parameter is used correctly
+        # prior to putatively setting in the sbatch call
+        if job.resources.get("slurm_extra"):
+            self.check_slurm_extra(job)
 
-        if not self.workflow.executor_settings.no_account:
-            call += self.get_account_arg(job)
+        job_params = {
+            "run_uuid": self.run_uuid,
+            "slurm_logfile": slurm_logfile,
+            "comment_str": comment_str,
+            "account": self.get_account_arg(job),
+            "partition": self.get_partition_arg(job),
+            "workdir": self.workflow.workdir_init,
+        }
 
-        call += self.get_partition_arg(job)
+        call = get_submit_command(job, job_params)
 
         if self.workflow.executor_settings.requeue:
             call += " --requeue"
 
         call += set_gres_string(job)
 
-        if job.resources.get("clusters"):
-            call += f" --clusters {job.resources.clusters}"
-
-        if job.resources.get("runtime"):
-            call += f" -t {job.resources.runtime}"
-        else:
+        if not job.resources.get("runtime"):
             self.logger.warning(
                 "No wall time information given. This might or might not "
                 "work on your cluster. "
@@ -245,28 +255,12 @@ class Executor(RemoteExecutor):
                 "default via --default-resources."
             )
 
-        if job.resources.get("constraint"):
-            call += f" -C '{job.resources.constraint}'"
-        if job.resources.get("mem_mb_per_cpu"):
-            call += f" --mem-per-cpu {job.resources.mem_mb_per_cpu}"
-        elif job.resources.get("mem_mb"):
-            call += f" --mem {job.resources.mem_mb}"
-        else:
+        if not job.resources.get("mem_mb_per_cpu") and not job.resources.get("mem_mb"):
             self.logger.warning(
                 "No job memory information ('mem_mb' or 'mem_mb_per_cpu') is given "
                 "- submitting without. This might or might not work on your cluster."
             )
 
-        if job.resources.get("nodes", False):
-            call += f" --nodes={job.resources.get('nodes', 1)}"
-
-        # fixes #40 - set ntasks regardless of mpi, because
-        # SLURM v22.05 will require it for all jobs
-        gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
-        if gpu_job:
-            call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
-        else:
-            call += f" --ntasks={job.resources.get('tasks', 1)}"
         # MPI job
         if job.resources.get("mpi", False):
             if not job.resources.get("tasks_per_node") and not job.resources.get(
@@ -278,19 +272,8 @@ class Executor(RemoteExecutor):
                     "Probably not what you want."
                 )
 
-        # we need to set cpus-per-task OR cpus-per-gpu, the function
-        # will return a string with the corresponding value
-        call += f" {get_cpu_setting(job, gpu_job)}"
-        if job.resources.get("slurm_extra"):
-            self.check_slurm_extra(job)
-            call += f" {job.resources.slurm_extra}"
-
         exec_job = self.format_job_exec(job)
 
-        # ensure that workdir is set correctly
-        # use short argument as this is the same in all slurm versions
-        # (see https://github.com/snakemake/snakemake/issues/2014)
-        call += f" -D {self.workflow.workdir_init}"
         # and finally the job to execute with all the snakemake parameters
         call += f' --wrap="{exec_job}"'
 
@@ -376,7 +359,11 @@ class Executor(RemoteExecutor):
 
         sacct_query_durations = []
 
-        status_attempts = 5
+        status_attempts = self.workflow.executor_settings.status_attempts
+        self.logger.debug(
+            f"Checking the status of {len(active_jobs)} active jobs "
+            f"with {status_attempts} attempts."
+        )
 
         active_jobs_ids = {job_info.external_jobid for job_info in active_jobs}
         active_jobs_seen_by_sacct = set()
@@ -510,7 +497,7 @@ We leave it to SLURM to resume your job(s)"""
                     self.next_seconds_between_status_checks + 10, max_sleep_time
                 )
             else:
-                self.next_seconds_between_status_checks = None
+                self.next_seconds_between_status_checks = 40
 
     def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]):
         # Cancel all active jobs.
@@ -570,10 +557,22 @@ We leave it to SLURM to resume your job(s)"""
                 for entry in csv.reader(StringIO(command_res), delimiter="|")
             }
         except subprocess.CalledProcessError as e:
-            self.logger.error(
-                f"The job status query failed with command: {command}\n"
-                f"Error message: {e.stderr.strip()}\n"
-            )
+            error_message = e.stderr.strip()
+            if "slurm_persist_conn_open_without_init" in error_message:
+                self.logger.warning(
+                    "The SLURM database might not be available ... "
+                    f"Error message: '{error_message}'"
+                    "This error message indicates that the SLURM database is currently "
+                    "not available. This is not an error of the Snakemake plugin, "
+                    "but some kind of server issue. "
+                    "Please consult with your HPC provider."
+                )
+            else:
+                self.logger.error(
+                    f"The job status query failed with command '{command}'"
+                    f"Error message: '{error_message}'"
+                    "This error message is not expected, please report it back to us."
+                )
             pass
 
         return (res, query_duration)
@@ -684,7 +683,7 @@ We leave it to SLURM to resume your job(s)"""
             )
             return ""
 
-        if account not in accounts:
+        if account.lower() not in accounts:
             raise WorkflowError(
                 f"The given account {account} appears to be invalid. Available "
                 f"accounts:\n{', '.join(accounts)}"

snakemake_executor_plugin_slurm/submit_string.py

@@ -0,0 +1,72 @@
+from snakemake_executor_plugin_slurm_jobstep import get_cpu_setting
+from types import SimpleNamespace
+
+
+def get_submit_command(job, params):
+    """
+    Return the submit command for the job.
+    """
+    # Convert params dict to a SimpleNamespace for attribute-style access
+    params = SimpleNamespace(**params)
+
+    call = (
+        f"sbatch "
+        f"--parsable "
+        f"--job-name {params.run_uuid} "
+        f'--output "{params.slurm_logfile}" '
+        f"--export=ALL "
+        f'--comment "{params.comment_str}"'
+    )
+
+    # No accout or partition checking is required, here.
+    # Checking is done in the submit function.
+
+    # here, only the string is used, as it already contains
+    # '-A {account_name}'
+    call += f" {params.account}"
+    # here, only the string is used, as it already contains
+    # '- p {partition_name}'
+    call += f" {params.partition}"
+
+    if job.resources.get("clusters"):
+        call += f" --clusters {job.resources.clusters}"
+
+    if job.resources.get("runtime"):
+        call += f" -t {job.resources.runtime}"
+
+    if job.resources.get("constraint") or isinstance(
+        job.resources.get("constraint"), str
+    ):
+        call += f" -C '{job.resources.get('constraint')}'"
+
+    if job.resources.get("qos") or isinstance(job.resources.get("qos"), str):
+        call += f" --qos='{job.resources.qos}'"
+
+    if job.resources.get("mem_mb_per_cpu"):
+        call += f" --mem-per-cpu {job.resources.mem_mb_per_cpu}"
+    elif job.resources.get("mem_mb"):
+        call += f" --mem {job.resources.mem_mb}"
+
+    if job.resources.get("nodes", False):
+        call += f" --nodes={job.resources.get('nodes', 1)}"
+
+    # fixes #40 - set ntasks regardless of mpi, because
+    # SLURM v22.05 will require it for all jobs
+    gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
+    if gpu_job:
+        call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
+    else:
+        call += f" --ntasks={job.resources.get('tasks', 1)}"
+
+    # we need to set cpus-per-task OR cpus-per-gpu, the function
+    # will return a string with the corresponding value
+    call += f" {get_cpu_setting(job, gpu_job)}"
+    if job.resources.get("slurm_extra"):
+        call += f" {job.resources.slurm_extra}"
+
+    # ensure that workdir is set correctly
+    # use short argument as this is the same in all slurm versions
+    # (see https://github.com/snakemake/snakemake/issues/2014)
+    call += f" -D '{params.workdir}'"
+
+    return call

{snakemake_executor_plugin_slurm-1.0.1.dist-info → snakemake_executor_plugin_slurm-1.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 1.0.1
+Version: 1.2.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 License: MIT
 Keywords: snakemake,plugin,executor,cluster,slurm

snakemake_executor_plugin_slurm-1.2.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+snakemake_executor_plugin_slurm/__init__.py,sha256=EqMKNkKYVFeDfw2pwCnFKYxgKOGJazlDm658wvFvQN0,30942
+snakemake_executor_plugin_slurm/submit_string.py,sha256=sXzMm5SVNQ4upIOcsIZjUqj7khnG-lieo5yJSSus5sc,2483
+snakemake_executor_plugin_slurm/utils.py,sha256=ZzXiXFDVLs15PLJnDP0eq98fNCtzlLbhtT03ec8Ou34,3578
+snakemake_executor_plugin_slurm-1.2.0.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
+snakemake_executor_plugin_slurm-1.2.0.dist-info/METADATA,sha256=rOpk-4_-aw3w-2X0POSy6rAvFZnPfzArN6MT9CuUxwA,1360
+snakemake_executor_plugin_slurm-1.2.0.dist-info/WHEEL,sha256=fGIA9gx4Qxk2KDKeNJCbOEwSrmLtjWCwzBz351GyrPQ,88
+snakemake_executor_plugin_slurm-1.2.0.dist-info/RECORD,,

{snakemake_executor_plugin_slurm-1.0.1.dist-info → snakemake_executor_plugin_slurm-1.2.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.1.1
+Generator: poetry-core 2.1.2
 Root-Is-Purelib: true
 Tag: py3-none-any

snakemake_executor_plugin_slurm-1.0.1.dist-info/RECORD

@@ -1,6 +0,0 @@
-snakemake_executor_plugin_slurm/__init__.py,sha256=d9aiBqYfhZY54ooqiawCQ67Kv2cFVpUrLCtSAjFvr6c,30722
-snakemake_executor_plugin_slurm/utils.py,sha256=ZzXiXFDVLs15PLJnDP0eq98fNCtzlLbhtT03ec8Ou34,3578
-snakemake_executor_plugin_slurm-1.0.1.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
-snakemake_executor_plugin_slurm-1.0.1.dist-info/METADATA,sha256=BK6xoB4FHYho7p5mxYUOlsp2T8dipyuUIV21b0sLVOE,1360
-snakemake_executor_plugin_slurm-1.0.1.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
-snakemake_executor_plugin_slurm-1.0.1.dist-info/RECORD,,