snakemake-executor-plugin-slurm 1.0.0__tar.gz → 1.1.0__tar.gz

{snakemake_executor_plugin_slurm-1.0.0 → snakemake_executor_plugin_slurm-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 1.0.0
+Version: 1.1.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 License: MIT
 Keywords: snakemake,plugin,executor,cluster,slurm

{snakemake_executor_plugin_slurm-1.0.0 → snakemake_executor_plugin_slurm-1.1.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "snakemake-executor-plugin-slurm"
-version = "1.0.0"
+version = "1.1.0"
 description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
 authors = [
     "Christian Meesters <meesters@uni-mainz.de>",

{snakemake_executor_plugin_slurm-1.0.0 → snakemake_executor_plugin_slurm-1.1.0}/snakemake_executor_plugin_slurm/__init__.py

@@ -74,6 +74,18 @@ class ExecutorSettings(ExecutorSettingsBase):
             "required": False,
         },
     )
+    status_attempts: Optional[int] = field(
+        default=5,
+        metadata={
+            "help": "Defines the number of attempts to query the status of "
+            "all active jobs. If the status query fails, the next attempt "
+            "will be performed after the next status check interval."
+            "The default is 5 status attempts before giving up. The maximum "
+            "time between status checks is 180 seconds.",
+            "env_var": False,
+            "required": False,
+        },
+    )
     requeue: bool = field(
         default=False,
         metadata={
@@ -85,6 +97,15 @@ class ExecutorSettings(ExecutorSettingsBase):
             "required": False,
         },
     )
+    no_account: bool = field(
+        default=False,
+        metadata={
+            "help": "Do not use any account for submission. "
+            "This flag has no effect, if not set.",
+            "env_var": False,
+            "required": False,
+        },
+    )
 
 
 # Required:
@@ -213,7 +234,9 @@ class Executor(RemoteExecutor):
             f"--comment '{comment_str}'"
         )
 
-        call += self.get_account_arg(job)
+        if not self.workflow.executor_settings.no_account:
+            call += self.get_account_arg(job)
+
         call += self.get_partition_arg(job)
 
         if self.workflow.executor_settings.requeue:
@@ -365,7 +388,11 @@ class Executor(RemoteExecutor):
 
         sacct_query_durations = []
 
-        status_attempts = 5
+        status_attempts = self.workflow.executor_settings.status_attempts
+        self.logger.debug(
+            f"Checking the status of {len(active_jobs)} active jobs "
+            f"with {status_attempts} attempts."
+        )
 
         active_jobs_ids = {job_info.external_jobid for job_info in active_jobs}
         active_jobs_seen_by_sacct = set()
@@ -499,7 +526,7 @@ We leave it to SLURM to resume your job(s)"""
                     self.next_seconds_between_status_checks + 10, max_sleep_time
                 )
             else:
-                self.next_seconds_between_status_checks = None
+                self.next_seconds_between_status_checks = 40
 
     def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]):
         # Cancel all active jobs.
@@ -559,10 +586,22 @@ We leave it to SLURM to resume your job(s)"""
                 for entry in csv.reader(StringIO(command_res), delimiter="|")
             }
         except subprocess.CalledProcessError as e:
-            self.logger.error(
-                f"The job status query failed with command: {command}\n"
-                f"Error message: {e.stderr.strip()}\n"
-            )
+            error_message = e.stderr.strip()
+            if "slurm_persist_conn_open_without_init" in error_message:
+                self.logger.warning(
+                    "The SLURM database might not be available ... "
+                    f"Error message: '{error_message}'"
+                    "This error message indicates that the SLURM database is currently "
+                    "not available. This is not an error of the Snakemake plugin, "
+                    "but some kind of server issue. "
+                    "Please consult with your HPC provider."
+                )
+            else:
+                self.logger.error(
+                    f"The job status query failed with command '{command}'"
+                    f"Error message: '{error_message}'"
+                    "This error message is not expected, please report it back to us."
+                )
             pass
 
         return (res, query_duration)
@@ -634,35 +673,45 @@ We leave it to SLURM to resume your job(s)"""
         """
         tests whether the given account is registered, raises an error, if not
         """
-        cmd = f'sacctmgr -n -s list user "{os.environ["USER"]}" format=account%256'
+        cmd = "sshare -U --format Account --noheader"
         try:
             accounts = subprocess.check_output(
                 cmd, shell=True, text=True, stderr=subprocess.PIPE
             )
         except subprocess.CalledProcessError as e:
-            sacctmgr_report = (
-                "Unable to test the validity of the given or guessed "
-                f"SLURM account '{account}' with sacctmgr: {e.stderr}."
+            sshare_report = (
+                "Unable to test the validity of the given or guessed"
+                f" SLURM account '{account}' with sshare: {e.stderr}."
             )
+            accounts = ""
+
+        if not accounts.strip():
+            cmd = f'sacctmgr -n -s list user "{os.environ["USER"]}" format=account%256'
             try:
-                cmd = "sshare -U --format Account --noheader"
                 accounts = subprocess.check_output(
                     cmd, shell=True, text=True, stderr=subprocess.PIPE
                 )
-            except subprocess.CalledProcessError as e2:
-                sshare_report = (
-                    "Unable to test the validity of the given or guessed"
-                    f" SLURM account '{account}' with sshare: {e2.stderr}."
+            except subprocess.CalledProcessError as e:
+                sacctmgr_report = (
+                    "Unable to test the validity of the given or guessed "
+                    f"SLURM account '{account}' with sacctmgr: {e.stderr}."
                 )
                 raise WorkflowError(
-                    f"The 'sacctmgr' reported: '{sacctmgr_report}' "
-                    f"and likewise 'sshare' reported: '{sshare_report}'."
+                    f"The 'sshare' reported: '{sshare_report}' "
+                    f"and likewise 'sacctmgr' reported: '{sacctmgr_report}'."
                 )
 
         # The set() has been introduced during review to eliminate
         # duplicates. They are not harmful, but disturbing to read.
         accounts = set(_.strip() for _ in accounts.split("\n") if _)
 
+        if not accounts:
+            self.logger.warning(
+                f"Both 'sshare' and 'sacctmgr' returned empty results for account "
+                f"'{account}'. Proceeding without account validation."
+            )
+            return ""
+
         if account not in accounts:
             raise WorkflowError(
                 f"The given account {account} appears to be invalid. Available "