snakemake-executor-plugin-slurm 0.11.1__py3-none-any.whl → 0.12.0__py3-none-any.whl

snakemake_executor_plugin_slurm/__init__.py

@@ -3,9 +3,11 @@ __copyright__ = "Copyright 2023, David Lähnemann, Johannes Köster, Christian M
 __email__ = "johannes.koester@uni-due.de"
 __license__ = "MIT"
 
+import atexit
 import csv
 from io import StringIO
 import os
+from pathlib import Path
 import re
 import shlex
 import subprocess
@@ -26,18 +28,48 @@ from snakemake_interface_executor_plugins.jobs import (
 from snakemake_interface_common.exceptions import WorkflowError
 from snakemake_executor_plugin_slurm_jobstep import get_cpus_per_task
 
-from .utils import delete_slurm_environment
+from .utils import delete_slurm_environment, delete_empty_dirs
 
 
 @dataclass
 class ExecutorSettings(ExecutorSettingsBase):
+    logdir: Optional[Path] = field(
+        default=None,
+        metadata={
+            "help": "Per default the SLURM log directory is relative to "
+            "the working directory."
+            "This flag allows to set an alternative directory.",
+            "env_var": False,
+            "required": False,
+        },
+    )
+    keep_successful_logs: bool = field(
+        default=False,
+        metadata={
+            "help": "Per default SLURM log files will be deleted upon sucessful "
+            "completion of a job. Whenever a SLURM job fails, its log "
+            "file will be preserved. "
+            "This flag allows to keep all SLURM log files, even those "
+            "of successful jobs.",
+            "env_var": False,
+            "required": False,
+        },
+    )
+    delete_logfiles_older_than: Optional[int] = field(
+        default=10,
+        metadata={
+            "help": "Per default SLURM log files in the SLURM log directory "
+            "of a workflow will be deleted after 10 days. For this, "
+            "best leave the default log directory unaltered. "
+            "Setting this flag allows to change this behaviour. "
+            "If set to <=0, no old files will be deleted. ",
+        },
+    )
     init_seconds_before_status_checks: Optional[int] = field(
         default=40,
         metadata={
-            "help": """
-                    Defines the time in seconds before the first status
-                    check is performed after job submission.
-                    """,
+            "help": "Defines the time in seconds before the first status "
+            "check is performed after job submission.",
             "env_var": False,
             "required": False,
         },
@@ -45,11 +77,10 @@ class ExecutorSettings(ExecutorSettingsBase):
     requeue: bool = field(
         default=False,
         metadata={
-            "help": """
-                    Allow requeuing preempted of failed jobs,
-                    if no cluster default. Results in `sbatch ... --requeue ...`
-                    This flag has no effect, if not set.
-                    """,
+            "help": "Allow requeuing preempted of failed jobs, "
+            "if no cluster default. Results in "
+            "`sbatch ... --requeue ...` "
+            "This flag has no effect, if not set.",
             "env_var": False,
             "required": False,
         },
@@ -91,6 +122,32 @@ class Executor(RemoteExecutor):
         self._fallback_account_arg = None
         self._fallback_partition = None
         self._preemption_warning = False  # no preemption warning has been issued
+        self.slurm_logdir = None
+        atexit.register(self.clean_old_logs)
+
+    def clean_old_logs(self) -> None:
+        """Delete files older than specified age from the SLURM log directory."""
+        # shorthands:
+        age_cutoff = self.workflow.executor_settings.delete_logfiles_older_than
+        keep_all = self.workflow.executor_settings.keep_successful_logs
+        if age_cutoff <= 0 or keep_all:
+            return
+        cutoff_secs = age_cutoff * 86400
+        current_time = time.time()
+        self.logger.info(f"Cleaning up log files older than {age_cutoff} day(s)")
+        for path in self.slurm_logdir.rglob("*.log"):
+            if path.is_file():
+                try:
+                    file_age = current_time - path.stat().st_mtime
+                    if file_age > cutoff_secs:
+                        path.unlink()
+                except (OSError, FileNotFoundError) as e:
+                    self.logger.warning(f"Could not delete logfile {path}: {e}")
+        # we need a 2nd iteration to remove putatively empty directories
+        try:
+            delete_empty_dirs(self.slurm_logdir)
+        except (OSError, FileNotFoundError) as e:
+            self.logger.warning(f"Could not delete empty directory {path}: {e}")
 
     def warn_on_jobcontext(self, done=None):
         if not done:
@@ -123,18 +180,22 @@ class Executor(RemoteExecutor):
         except AttributeError:
             wildcard_str = ""
 
-        slurm_logfile = os.path.abspath(
-            f".snakemake/slurm_logs/{group_or_rule}/{wildcard_str}/%j.log"
+        self.slurm_logdir = (
+            Path(self.workflow.executor_settings.logdir)
+            if self.workflow.executor_settings.logdir
+            else Path(".snakemake/slurm_logs").resolve()
         )
-        logdir = os.path.dirname(slurm_logfile)
+
+        self.slurm_logdir.mkdir(parents=True, exist_ok=True)
+        slurm_logfile = self.slurm_logdir / group_or_rule / wildcard_str / "%j.log"
+        slurm_logfile.parent.mkdir(parents=True, exist_ok=True)
         # this behavior has been fixed in slurm 23.02, but there might be plenty of
         # older versions around, hence we should rather be conservative here.
-        assert "%j" not in logdir, (
+        assert "%j" not in str(self.slurm_logdir), (
             "bug: jobid placeholder in parent dir of logfile. This does not work as "
             "we have to create that dir before submission in order to make sbatch "
             "happy. Otherwise we get silent fails without logfiles being created."
         )
-        os.makedirs(logdir, exist_ok=True)
 
         # generic part of a submission string:
         # we use a run_uuid as the job-name, to allow `--name`-based
@@ -217,12 +278,26 @@ class Executor(RemoteExecutor):
 
         self.logger.debug(f"sbatch call: {call}")
         try:
-            out = subprocess.check_output(
-                call, shell=True, text=True, stderr=subprocess.STDOUT
-            ).strip()
+            process = subprocess.Popen(
+                call,
+                shell=True,
+                text=True,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            out, err = process.communicate()
+            if process.returncode != 0:
+                raise subprocess.CalledProcessError(
+                    process.returncode, call, output=err
+                )
         except subprocess.CalledProcessError as e:
             raise WorkflowError(
-                f"SLURM job submission failed. The error message was {e.output}"
+                f"SLURM sbatch failed. The error message was {e.output}"
+            )
+        # any other error message indicating failure?
+        if "submission failed" in err:
+            raise WorkflowError(
+                f"SLURM job submission failed. The error message was {err}"
             )
 
         # multicluster submissions yield submission infos like
@@ -230,8 +305,12 @@ class Executor(RemoteExecutor):
         # --parsable option it simply yields "<id>;<name>".
         # To extract the job id we split by semicolon and take the first element
         # (this also works if no cluster name was provided)
-        slurm_jobid = out.split(";")[0]
-        slurm_logfile = slurm_logfile.replace("%j", slurm_jobid)
+        slurm_jobid = out.strip().split(";")[0]
+        if not slurm_jobid:
+            raise WorkflowError("Failed to retrieve SLURM job ID from sbatch output.")
+        slurm_logfile = slurm_logfile.with_name(
+            slurm_logfile.name.replace("%j", slurm_jobid)
+        )
         self.logger.info(
             f"Job {job.jobid} has been submitted with SLURM jobid {slurm_jobid} "
             f"(log: {slurm_logfile})."
@@ -364,6 +443,19 @@ class Executor(RemoteExecutor):
                     self.report_job_success(j)
                     any_finished = True
                     active_jobs_seen_by_sacct.remove(j.external_jobid)
+                    if not self.workflow.executor_settings.keep_successful_logs:
+                        self.logger.debug(
+                            "removing log for successful job "
+                            f"with SLURM ID '{j.external_jobid}'"
+                        )
+                        try:
+                            if j.aux["slurm_logfile"].exists():
+                                j.aux["slurm_logfile"].unlink()
+                        except (OSError, FileNotFoundError) as e:
+                            self.logger.warning(
+                                "Could not remove log file"
+                                f" {j.aux['slurm_logfile']}: {e}"
+                            )
                 elif status == "PREEMPTED" and not self._preemption_warning:
                     self._preemption_warning = True
                     self.logger.warning(
@@ -388,7 +480,9 @@ We leave it to SLURM to resume your job(s)"""
                         # with a new sentence
                         f"'{status}'. "
                     )
-                    self.report_job_error(j, msg=msg, aux_logs=[j.aux["slurm_logfile"]])
+                    self.report_job_error(
+                        j, msg=msg, aux_logs=[j.aux["slurm_logfile"]._str]
+                    )
                     active_jobs_seen_by_sacct.remove(j.external_jobid)
                 else:  # still running?
                     yield j

snakemake_executor_plugin_slurm/utils.py

@@ -1,6 +1,7 @@
 # utility functions for the SLURM executor plugin
 
 import os
+from pathlib import Path
 
 
 def delete_slurm_environment():
@@ -14,3 +15,28 @@ def delete_slurm_environment():
     for var in os.environ:
         if var.startswith("SLURM_"):
             del os.environ[var]
+
+
+def delete_empty_dirs(path: Path) -> None:
+    """
+    Function to delete all empty directories in a given path.
+    This is needed to clean up the working directory after
+    a job has sucessfully finished. This function is needed because
+    the shutil.rmtree() function does not delete empty
+    directories.
+    """
+    if not path.is_dir():
+        return
+
+    # Process subdirectories first (bottom-up)
+    for child in path.iterdir():
+        if child.is_dir():
+            delete_empty_dirs(child)
+
+    try:
+        # Check if directory is now empty after processing children
+        if not any(path.iterdir()):
+            path.rmdir()
+    except (OSError, FileNotFoundError) as e:
+        # Provide more context in the error message
+        raise OSError(f"Failed to remove empty directory {path}: {e}") from e

{snakemake_executor_plugin_slurm-0.11.1.dist-info → snakemake_executor_plugin_slurm-0.12.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 0.11.1
+Version: 0.12.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 Home-page: https://github.com/snakemake/snakemake-executor-plugin-slurm
 License: MIT

snakemake_executor_plugin_slurm-0.12.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+snakemake_executor_plugin_slurm/__init__.py,sha256=HhAOwrgUp31fM9ciKOhU1HtY1zfdPj_yQbRu9CKj7vY,29029
+snakemake_executor_plugin_slurm/utils.py,sha256=JOpQaUviGz6SORrMUsVDrSHc0lH6qX_SM0eUjVbWgp0,1282
+snakemake_executor_plugin_slurm-0.12.0.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
+snakemake_executor_plugin_slurm-0.12.0.dist-info/METADATA,sha256=CkoWIpPni0VLr-EXTxv33UoVe8DQoL_PuLYfGgv5PmA,1432
+snakemake_executor_plugin_slurm-0.12.0.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+snakemake_executor_plugin_slurm-0.12.0.dist-info/RECORD,,

{snakemake_executor_plugin_slurm-0.11.1.dist-info → snakemake_executor_plugin_slurm-0.12.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.0.0
 Root-Is-Purelib: true
 Tag: py3-none-any

snakemake_executor_plugin_slurm-0.11.1.dist-info/RECORD

@@ -1,6 +0,0 @@
-snakemake_executor_plugin_slurm/__init__.py,sha256=FleQvFmJ6A2h2uCCTBD92T8cSb5Bcol-sgJ-ggs6ggA,24923
-snakemake_executor_plugin_slurm/utils.py,sha256=DuJdFJsAmvFsrnpyb8kMoqxTEEmTsEVxroDS1t9qOGw,434
-snakemake_executor_plugin_slurm-0.11.1.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
-snakemake_executor_plugin_slurm-0.11.1.dist-info/METADATA,sha256=hCt8Po7tX2EndsxLh-eDe6zoB6mgTn2OhVp50UIdApQ,1432
-snakemake_executor_plugin_slurm-0.11.1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-snakemake_executor_plugin_slurm-0.11.1.dist-info/RECORD,,