snakemake-executor-plugin-slurm 0.15.1__tar.gz → 1.0.1__tar.gz

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 0.15.1
+Version: 1.0.1
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 License: MIT
 Keywords: snakemake,plugin,executor,cluster,slurm
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: snakemake-executor-plugin-slurm-jobstep (>=0.2.0,<0.3.0)
+Requires-Dist: snakemake-executor-plugin-slurm-jobstep (>=0.3.0,<0.4.0)
 Requires-Dist: snakemake-interface-common (>=1.13.0,<2.0.0)
 Requires-Dist: snakemake-interface-executor-plugins (>=9.1.1,<10.0.0)
 Requires-Dist: throttler (>=1.2.2,<2.0.0)
@@ -25,3 +25,4 @@ Description-Content-Type: text/markdown
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/snakemake/snakemake-executor-plugin-slurm)
 
 For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html).
+

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.1}/README.md

@@ -2,4 +2,4 @@
 
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/snakemake/snakemake-executor-plugin-slurm)
 
-For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html).
+For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html).

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "snakemake-executor-plugin-slurm"
-version = "0.15.1"
+version = "1.0.1"
 description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
 authors = [
     "Christian Meesters <meesters@uni-mainz.de>",
@@ -17,7 +17,7 @@ keywords = ["snakemake", "plugin", "executor", "cluster", "slurm"]
 python = "^3.11"
 snakemake-interface-common = "^1.13.0"
 snakemake-interface-executor-plugins = "^9.1.1"
-snakemake-executor-plugin-slurm-jobstep = "^0.2.0"
+snakemake-executor-plugin-slurm-jobstep = "^0.3.0"
 throttler = "^1.2.2"
 
 [tool.poetry.group.dev.dependencies]

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.1}/snakemake_executor_plugin_slurm/__init__.py

@@ -26,9 +26,9 @@ from snakemake_interface_executor_plugins.jobs import (
     JobExecutorInterface,
 )
 from snakemake_interface_common.exceptions import WorkflowError
-from snakemake_executor_plugin_slurm_jobstep import get_cpus_per_task
+from snakemake_executor_plugin_slurm_jobstep import get_cpu_setting
 
-from .utils import delete_slurm_environment, delete_empty_dirs
+from .utils import delete_slurm_environment, delete_empty_dirs, set_gres_string
 
 
 @dataclass
@@ -85,6 +85,15 @@ class ExecutorSettings(ExecutorSettingsBase):
             "required": False,
         },
     )
+    no_account: bool = field(
+        default=False,
+        metadata={
+            "help": "Do not use any account for submission. "
+            "This flag has no effect, if not set.",
+            "env_var": False,
+            "required": False,
+        },
+    )
 
 
 # Required:
@@ -180,7 +189,9 @@ class Executor(RemoteExecutor):
         group_or_rule = f"group_{job.name}" if job.is_group() else f"rule_{job.name}"
 
         try:
-            wildcard_str = "_".join(job.wildcards) if job.wildcards else ""
+            wildcard_str = (
+                "_".join(job.wildcards).replace("/", "_") if job.wildcards else ""
+            )
         except AttributeError:
             wildcard_str = ""
 
@@ -211,12 +222,16 @@ class Executor(RemoteExecutor):
             f"--comment '{comment_str}'"
         )
 
-        call += self.get_account_arg(job)
+        if not self.workflow.executor_settings.no_account:
+            call += self.get_account_arg(job)
+
         call += self.get_partition_arg(job)
 
         if self.workflow.executor_settings.requeue:
             call += " --requeue"
 
+        call += set_gres_string(job)
+
         if job.resources.get("clusters"):
             call += f" --clusters {job.resources.clusters}"
 
@@ -247,7 +262,11 @@ class Executor(RemoteExecutor):
 
         # fixes #40 - set ntasks regardless of mpi, because
         # SLURM v22.05 will require it for all jobs
-        call += f" --ntasks={job.resources.get('tasks', 1)}"
+        gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
+        if gpu_job:
+            call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
+        else:
+            call += f" --ntasks={job.resources.get('tasks', 1)}"
         # MPI job
         if job.resources.get("mpi", False):
             if not job.resources.get("tasks_per_node") and not job.resources.get(
@@ -259,8 +278,9 @@ class Executor(RemoteExecutor):
                     "Probably not what you want."
                 )
 
-        call += f" --cpus-per-task={get_cpus_per_task(job)}"
-
+        # we need to set cpus-per-task OR cpus-per-gpu, the function
+        # will return a string with the corresponding value
+        call += f" {get_cpu_setting(job, gpu_job)}"
         if job.resources.get("slurm_extra"):
             self.check_slurm_extra(job)
             call += f" {job.resources.slurm_extra}"
@@ -625,35 +645,45 @@ We leave it to SLURM to resume your job(s)"""
         """
         tests whether the given account is registered, raises an error, if not
         """
-        cmd = f'sacctmgr -n -s list user "{os.environ["USER"]}" format=account%256'
+        cmd = "sshare -U --format Account --noheader"
         try:
             accounts = subprocess.check_output(
                 cmd, shell=True, text=True, stderr=subprocess.PIPE
             )
         except subprocess.CalledProcessError as e:
-            sacctmgr_report = (
-                "Unable to test the validity of the given or guessed "
-                f"SLURM account '{account}' with sacctmgr: {e.stderr}."
+            sshare_report = (
+                "Unable to test the validity of the given or guessed"
+                f" SLURM account '{account}' with sshare: {e.stderr}."
             )
+            accounts = ""
+
+        if not accounts.strip():
+            cmd = f'sacctmgr -n -s list user "{os.environ["USER"]}" format=account%256'
             try:
-                cmd = "sshare -U --format Account --noheader"
                 accounts = subprocess.check_output(
                     cmd, shell=True, text=True, stderr=subprocess.PIPE
                 )
-            except subprocess.CalledProcessError as e2:
-                sshare_report = (
-                    "Unable to test the validity of the given or guessed"
-                    f" SLURM account '{account}' with sshare: {e2.stderr}."
+            except subprocess.CalledProcessError as e:
+                sacctmgr_report = (
+                    "Unable to test the validity of the given or guessed "
+                    f"SLURM account '{account}' with sacctmgr: {e.stderr}."
                 )
                 raise WorkflowError(
-                    f"The 'sacctmgr' reported: '{sacctmgr_report}' "
-                    f"and likewise 'sshare' reported: '{sshare_report}'."
+                    f"The 'sshare' reported: '{sshare_report}' "
+                    f"and likewise 'sacctmgr' reported: '{sacctmgr_report}'."
                 )
 
         # The set() has been introduced during review to eliminate
         # duplicates. They are not harmful, but disturbing to read.
         accounts = set(_.strip() for _ in accounts.split("\n") if _)
 
+        if not accounts:
+            self.logger.warning(
+                f"Both 'sshare' and 'sacctmgr' returned empty results for account "
+                f"'{account}'. Proceeding without account validation."
+            )
+            return ""
+
         if account not in accounts:
             raise WorkflowError(
                 f"The given account {account} appears to be invalid. Available "

snakemake_executor_plugin_slurm-1.0.1/snakemake_executor_plugin_slurm/utils.py

@@ -0,0 +1,104 @@
+# utility functions for the SLURM executor plugin
+
+import os
+import re
+from pathlib import Path
+
+from snakemake_interface_executor_plugins.jobs import (
+    JobExecutorInterface,
+)
+from snakemake_interface_common.exceptions import WorkflowError
+
+
+def delete_slurm_environment():
+    """
+    Function to delete all environment variables
+    starting with 'SLURM_'. The parent shell will
+    still have this environment. This is needed to
+    submit within a SLURM job context to avoid
+    conflicting environments.
+    """
+    for var in os.environ:
+        if var.startswith("SLURM_"):
+            del os.environ[var]
+
+
+def delete_empty_dirs(path: Path) -> None:
+    """
+    Function to delete all empty directories in a given path.
+    This is needed to clean up the working directory after
+    a job has sucessfully finished. This function is needed because
+    the shutil.rmtree() function does not delete empty
+    directories.
+    """
+    if not path.is_dir():
+        return
+
+    # Process subdirectories first (bottom-up)
+    for child in path.iterdir():
+        if child.is_dir():
+            delete_empty_dirs(child)
+
+    try:
+        # Check if directory is now empty after processing children
+        if not any(path.iterdir()):
+            path.rmdir()
+    except (OSError, FileNotFoundError) as e:
+        # Provide more context in the error message
+        raise OSError(f"Failed to remove empty directory {path}: {e}") from e
+
+
+def set_gres_string(job: JobExecutorInterface) -> str:
+    """
+    Function to set the gres string for the SLURM job
+    based on the resources requested in the job.
+    """
+    # generic resources (GRES) arguments can be of type
+    # "string:int" or "string:string:int"
+    gres_re = re.compile(r"^[a-zA-Z0-9_]+(:[a-zA-Z0-9_]+)?:\d+$")
+    # gpu model arguments can be of type "string"
+    gpu_model_re = re.compile(r"^[a-zA-Z0-9_]+$")
+    # The Snakemake resources can be only be of type "int",
+    # hence no further regex is needed.
+
+    gpu_string = None
+    if job.resources.get("gpu"):
+        gpu_string = str(job.resources.get("gpu"))
+
+    gpu_model = None
+    if job.resources.get("gpu_model"):
+        gpu_model = job.resources.get("gpu_model")
+
+    # ensure that gres is not set, if gpu and gpu_model are set
+    if job.resources.get("gres") and gpu_string:
+        raise WorkflowError(
+            "GRES and GPU are set. Please only set one of them.", rule=job.rule
+        )
+    elif not job.resources.get("gres") and not gpu_model and not gpu_string:
+        return ""
+
+    if job.resources.get("gres"):
+        # Validate GRES format (e.g., "gpu:1", "gpu:tesla:2")
+        gres = job.resources.gres
+        if not gres_re.match(gres):
+            raise WorkflowError(
+                f"Invalid GRES format: {gres}. Expected format: "
+                "'<name>:<number>' or '<name>:<type>:<number>' "
+                "(e.g., 'gpu:1' or 'gpu:tesla:2')"
+            )
+        return f" --gres={job.resources.gres}"
+
+    if gpu_model and gpu_string:
+        # validate GPU model format
+        if not gpu_model_re.match(gpu_model):
+            raise WorkflowError(
+                f"Invalid GPU model format: {gpu_model}."
+                " Expected format: '<name>' (e.g., 'tesla')"
+            )
+        return f" --gpus={gpu_model}:{gpu_string}"
+    elif gpu_model and not gpu_string:
+        raise WorkflowError("GPU model is set, but no GPU number is given")
+    elif gpu_string:
+        # we assume here, that the validator ensures that the 'gpu_string'
+        # is an integer
+        return f" --gpus={gpu_string}"

snakemake_executor_plugin_slurm-0.15.1/snakemake_executor_plugin_slurm/utils.py

@@ -1,42 +0,0 @@
-# utility functions for the SLURM executor plugin
-
-import os
-from pathlib import Path
-
-
-def delete_slurm_environment():
-    """
-    Function to delete all environment variables
-    starting with 'SLURM_'. The parent shell will
-    still have this environment. This is needed to
-    submit within a SLURM job context to avoid
-    conflicting environments.
-    """
-    for var in os.environ:
-        if var.startswith("SLURM_"):
-            del os.environ[var]
-
-
-def delete_empty_dirs(path: Path) -> None:
-    """
-    Function to delete all empty directories in a given path.
-    This is needed to clean up the working directory after
-    a job has sucessfully finished. This function is needed because
-    the shutil.rmtree() function does not delete empty
-    directories.
-    """
-    if not path.is_dir():
-        return
-
-    # Process subdirectories first (bottom-up)
-    for child in path.iterdir():
-        if child.is_dir():
-            delete_empty_dirs(child)
-
-    try:
-        # Check if directory is now empty after processing children
-        if not any(path.iterdir()):
-            path.rmdir()
-    except (OSError, FileNotFoundError) as e:
-        # Provide more context in the error message
-        raise OSError(f"Failed to remove empty directory {path}: {e}") from e