snakemake-executor-plugin-slurm 0.15.1__tar.gz → 1.0.0__tar.gz

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 0.15.1
+Version: 1.0.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 License: MIT
 Keywords: snakemake,plugin,executor,cluster,slurm
@@ -12,7 +12,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: snakemake-executor-plugin-slurm-jobstep (>=0.2.0,<0.3.0)
+Requires-Dist: snakemake-executor-plugin-slurm-jobstep (>=0.3.0,<0.4.0)
 Requires-Dist: snakemake-interface-common (>=1.13.0,<2.0.0)
 Requires-Dist: snakemake-interface-executor-plugins (>=9.1.1,<10.0.0)
 Requires-Dist: throttler (>=1.2.2,<2.0.0)
@@ -25,3 +25,4 @@ Description-Content-Type: text/markdown
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/snakemake/snakemake-executor-plugin-slurm)
 
 For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html).
+

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.0}/README.md

@@ -2,4 +2,4 @@
 
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/#https://github.com/snakemake/snakemake-executor-plugin-slurm)
 
-For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html).
+For documentation, see the [Snakemake plugin catalog](https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html).

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "snakemake-executor-plugin-slurm"
-version = "0.15.1"
+version = "1.0.0"
 description = "A Snakemake executor plugin for submitting jobs to a SLURM cluster."
 authors = [
     "Christian Meesters <meesters@uni-mainz.de>",
@@ -17,7 +17,7 @@ keywords = ["snakemake", "plugin", "executor", "cluster", "slurm"]
 python = "^3.11"
 snakemake-interface-common = "^1.13.0"
 snakemake-interface-executor-plugins = "^9.1.1"
-snakemake-executor-plugin-slurm-jobstep = "^0.2.0"
+snakemake-executor-plugin-slurm-jobstep = "^0.3.0"
 throttler = "^1.2.2"
 
 [tool.poetry.group.dev.dependencies]

{snakemake_executor_plugin_slurm-0.15.1 → snakemake_executor_plugin_slurm-1.0.0}/snakemake_executor_plugin_slurm/__init__.py

@@ -26,9 +26,9 @@ from snakemake_interface_executor_plugins.jobs import (
     JobExecutorInterface,
 )
 from snakemake_interface_common.exceptions import WorkflowError
-from snakemake_executor_plugin_slurm_jobstep import get_cpus_per_task
+from snakemake_executor_plugin_slurm_jobstep import get_cpu_setting
 
-from .utils import delete_slurm_environment, delete_empty_dirs
+from .utils import delete_slurm_environment, delete_empty_dirs, set_gres_string
 
 
 @dataclass
@@ -180,7 +180,9 @@ class Executor(RemoteExecutor):
         group_or_rule = f"group_{job.name}" if job.is_group() else f"rule_{job.name}"
 
         try:
-            wildcard_str = "_".join(job.wildcards) if job.wildcards else ""
+            wildcard_str = (
+                "_".join(job.wildcards).replace("/", "_") if job.wildcards else ""
+            )
         except AttributeError:
             wildcard_str = ""
 
@@ -217,6 +219,8 @@ class Executor(RemoteExecutor):
         if self.workflow.executor_settings.requeue:
             call += " --requeue"
 
+        call += set_gres_string(job)
+
         if job.resources.get("clusters"):
             call += f" --clusters {job.resources.clusters}"
 
@@ -247,7 +251,11 @@ class Executor(RemoteExecutor):
 
         # fixes #40 - set ntasks regardless of mpi, because
         # SLURM v22.05 will require it for all jobs
-        call += f" --ntasks={job.resources.get('tasks', 1)}"
+        gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
+        if gpu_job:
+            call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
+        else:
+            call += f" --ntasks={job.resources.get('tasks', 1)}"
         # MPI job
         if job.resources.get("mpi", False):
             if not job.resources.get("tasks_per_node") and not job.resources.get(
@@ -259,8 +267,9 @@ class Executor(RemoteExecutor):
                     "Probably not what you want."
                 )
 
-        call += f" --cpus-per-task={get_cpus_per_task(job)}"
-
+        # we need to set cpus-per-task OR cpus-per-gpu, the function
+        # will return a string with the corresponding value
+        call += f" {get_cpu_setting(job, gpu_job)}"
         if job.resources.get("slurm_extra"):
             self.check_slurm_extra(job)
             call += f" {job.resources.slurm_extra}"

snakemake_executor_plugin_slurm-1.0.0/snakemake_executor_plugin_slurm/utils.py

@@ -0,0 +1,104 @@
+# utility functions for the SLURM executor plugin
+
+import os
+import re
+from pathlib import Path
+
+from snakemake_interface_executor_plugins.jobs import (
+    JobExecutorInterface,
+)
+from snakemake_interface_common.exceptions import WorkflowError
+
+
+def delete_slurm_environment():
+    """
+    Function to delete all environment variables
+    starting with 'SLURM_'. The parent shell will
+    still have this environment. This is needed to
+    submit within a SLURM job context to avoid
+    conflicting environments.
+    """
+    for var in os.environ:
+        if var.startswith("SLURM_"):
+            del os.environ[var]
+
+
+def delete_empty_dirs(path: Path) -> None:
+    """
+    Function to delete all empty directories in a given path.
+    This is needed to clean up the working directory after
+    a job has sucessfully finished. This function is needed because
+    the shutil.rmtree() function does not delete empty
+    directories.
+    """
+    if not path.is_dir():
+        return
+
+    # Process subdirectories first (bottom-up)
+    for child in path.iterdir():
+        if child.is_dir():
+            delete_empty_dirs(child)
+
+    try:
+        # Check if directory is now empty after processing children
+        if not any(path.iterdir()):
+            path.rmdir()
+    except (OSError, FileNotFoundError) as e:
+        # Provide more context in the error message
+        raise OSError(f"Failed to remove empty directory {path}: {e}") from e
+
+
+def set_gres_string(job: JobExecutorInterface) -> str:
+    """
+    Function to set the gres string for the SLURM job
+    based on the resources requested in the job.
+    """
+    # generic resources (GRES) arguments can be of type
+    # "string:int" or "string:string:int"
+    gres_re = re.compile(r"^[a-zA-Z0-9_]+(:[a-zA-Z0-9_]+)?:\d+$")
+    # gpu model arguments can be of type "string"
+    gpu_model_re = re.compile(r"^[a-zA-Z0-9_]+$")
+    # The Snakemake resources can be only be of type "int",
+    # hence no further regex is needed.
+
+    gpu_string = None
+    if job.resources.get("gpu"):
+        gpu_string = str(job.resources.get("gpu"))
+
+    gpu_model = None
+    if job.resources.get("gpu_model"):
+        gpu_model = job.resources.get("gpu_model")
+
+    # ensure that gres is not set, if gpu and gpu_model are set
+    if job.resources.get("gres") and gpu_string:
+        raise WorkflowError(
+            "GRES and GPU are set. Please only set one of them.", rule=job.rule
+        )
+    elif not job.resources.get("gres") and not gpu_model and not gpu_string:
+        return ""
+
+    if job.resources.get("gres"):
+        # Validate GRES format (e.g., "gpu:1", "gpu:tesla:2")
+        gres = job.resources.gres
+        if not gres_re.match(gres):
+            raise WorkflowError(
+                f"Invalid GRES format: {gres}. Expected format: "
+                "'<name>:<number>' or '<name>:<type>:<number>' "
+                "(e.g., 'gpu:1' or 'gpu:tesla:2')"
+            )
+        return f" --gres={job.resources.gres}"
+
+    if gpu_model and gpu_string:
+        # validate GPU model format
+        if not gpu_model_re.match(gpu_model):
+            raise WorkflowError(
+                f"Invalid GPU model format: {gpu_model}."
+                " Expected format: '<name>' (e.g., 'tesla')"
+            )
+        return f" --gpus={gpu_model}:{gpu_string}"
+    elif gpu_model and not gpu_string:
+        raise WorkflowError("GPU model is set, but no GPU number is given")
+    elif gpu_string:
+        # we assume here, that the validator ensures that the 'gpu_string'
+        # is an integer
+        return f" --gpus={gpu_string}"

snakemake_executor_plugin_slurm-0.15.1/snakemake_executor_plugin_slurm/utils.py

@@ -1,42 +0,0 @@
-# utility functions for the SLURM executor plugin
-
-import os
-from pathlib import Path
-
-
-def delete_slurm_environment():
-    """
-    Function to delete all environment variables
-    starting with 'SLURM_'. The parent shell will
-    still have this environment. This is needed to
-    submit within a SLURM job context to avoid
-    conflicting environments.
-    """
-    for var in os.environ:
-        if var.startswith("SLURM_"):
-            del os.environ[var]
-
-
-def delete_empty_dirs(path: Path) -> None:
-    """
-    Function to delete all empty directories in a given path.
-    This is needed to clean up the working directory after
-    a job has sucessfully finished. This function is needed because
-    the shutil.rmtree() function does not delete empty
-    directories.
-    """
-    if not path.is_dir():
-        return
-
-    # Process subdirectories first (bottom-up)
-    for child in path.iterdir():
-        if child.is_dir():
-            delete_empty_dirs(child)
-
-    try:
-        # Check if directory is now empty after processing children
-        if not any(path.iterdir()):
-            path.rmdir()
-    except (OSError, FileNotFoundError) as e:
-        # Provide more context in the error message
-        raise OSError(f"Failed to remove empty directory {path}: {e}") from e