snakemake-executor-plugin-slurm 1.6.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

snakemake_executor_plugin_slurm/__init__.py

@@ -140,6 +140,14 @@ class ExecutorSettings(ExecutorSettingsBase):
             "This flag has no effect, if not set.",
         },
     )
+    qos: Optional[str] = field(
+        default=None,
+        metadata={
+            "help": "If set, the given QoS will be used for job submission.",
+            "env_var": False,
+            "required": False,
+        },
+    )
     reservation: Optional[str] = field(
         default=None,
         metadata={
@@ -309,6 +317,9 @@ class Executor(RemoteExecutor):
         if self.workflow.executor_settings.requeue:
             call += " --requeue"
 
+        if self.workflow.executor_settings.qos:
+            call += f" --qos={self.workflow.executor_settings.qos}"
+
         if self.workflow.executor_settings.reservation:
             call += f" --reservation={self.workflow.executor_settings.reservation}"
 
@@ -328,13 +339,6 @@ class Executor(RemoteExecutor):
                 "- submitting without. This might or might not work on your cluster."
             )
 
-            # fixes #40 - set ntasks regardless of mpi, because
-            # SLURM v22.05 introduced the requirement for all jobs
-            gpu_job = job.resources.get("gpu") or "gpu" in job.resources.get("gres", "")
-            if gpu_job:
-                call += f" --ntasks-per-gpu={job.resources.get('tasks', 1)}"
-            else:
-                call += f" --ntasks={job.resources.get('tasks', 1)}"
         # MPI job
         if job.resources.get("mpi", False):
             if not job.resources.get("tasks_per_node") and not job.resources.get(

snakemake_executor_plugin_slurm/efficiency_report.py

@@ -55,15 +55,10 @@ def parse_reqmem(reqmem, number_of_nodes=1):
     return 0
 
 
-def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
-    """
-    Fetch sacct job data for a Snakemake workflow
-    and compute efficiency metrics.
-    """
+def get_sacct_data(run_uuid, logger):
+    """Fetch raw sacct data for a workflow."""
     cmd = f"sacct --name={run_uuid} --parsable2 --noheader"
-    cmd += (
-        " --format=JobID,JobName,Comment,Elapsed,TotalCPU," "NNodes,NCPUS,MaxRSS,ReqMem"
-    )
+    cmd += " --format=JobID,JobName,Comment,Elapsed,TotalCPU,NNodes,NCPUS,MaxRSS,ReqMem"
 
     try:
         result = subprocess.run(
@@ -74,12 +69,14 @@ def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
             logger.warning(f"No job data found for workflow {run_uuid}.")
             return None
         lines = raw.split("\n")
+        return lines
 
     except subprocess.CalledProcessError:
         logger.error(f"Failed to retrieve job data for workflow {run_uuid}.")
         return None
 
-    # Convert to DataFrame
+
+def parse_sacct_data(lines, e_threshold, run_uuid, logger):
     df = pd.DataFrame(
         (line.split("|") for line in lines),
         columns=[
@@ -120,13 +117,6 @@ def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
     df["Elapsed_sec"] = df["Elapsed"].apply(time_to_seconds)
     df["TotalCPU_sec"] = df["TotalCPU"].apply(time_to_seconds)
 
-    # Compute CPU efficiency
-    df["CPU Efficiency (%)"] = (
-        df["TotalCPU_sec"]
-        / (df["Elapsed_sec"].clip(lower=1) * df["NCPUS"].clip(lower=1))
-    ) * 100
-    df.replace([np.inf, -np.inf], 0, inplace=True)
-
     # Convert MaxRSS
     df["MaxRSS_MB"] = df["MaxRSS"].apply(parse_maxrss)
 
@@ -134,6 +124,37 @@ def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
     df["RequestedMem_MB"] = df.apply(
         lambda row: parse_reqmem(row["ReqMem"], row["NNodes"]), axis=1
     )
+
+    # Drop all rows containing "batch" or "extern" as job names
+    df = df[~df["JobName"].str.contains("batch|extern", na=False)]
+
+    # Extract main job ID for grouping
+    df["MainJobID"] = df["JobID"].str.extract(r"^(\d+)", expand=False)
+
+    # Separate main jobs and job steps
+    main_jobs = df[~df["JobID"].str.contains(r"\.\d+", regex=True)].copy()
+    job_steps = df[df["JobID"].str.contains(r"\.\d+", regex=True)].copy()
+
+    # Create maps from main jobs for inheritance
+    if not nocomment:
+        rule_name_map = main_jobs.set_index("MainJobID")["RuleName"].to_dict()
+    mem_map = main_jobs.set_index("MainJobID")["RequestedMem_MB"].to_dict()
+
+    # Inherit data from main jobs to job steps
+    if not nocomment:
+        job_steps["RuleName"] = job_steps["MainJobID"].map(rule_name_map).fillna("")
+    job_steps["RequestedMem_MB"] = job_steps["MainJobID"].map(mem_map).fillna(0)
+
+    # Use job steps as the final dataset (they have the actual resource usage)
+    df = job_steps.copy()
+
+    # Compute CPU efficiency
+    df["CPU Efficiency (%)"] = (
+        df["TotalCPU_sec"]
+        / (df["Elapsed_sec"].clip(lower=1) * df["NCPUS"].clip(lower=1))
+    ) * 100
+    df.replace([np.inf, -np.inf], 0, inplace=True)
+
     df["Memory Usage (%)"] = df.apply(
         lambda row: (
             (row["MaxRSS_MB"] / row["RequestedMem_MB"] * 100)
@@ -145,9 +166,6 @@ def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
 
     df["Memory Usage (%)"] = df["Memory Usage (%)"].fillna(0).round(2)
 
-    # Drop all rows containing "batch" or "extern" as job names
-    df = df[~df["JobName"].str.contains("batch|extern", na=False)]
-
     # Log warnings for low efficiency
     for _, row in df.iterrows():
         if row["CPU Efficiency (%)"] < e_threshold:
@@ -164,6 +182,20 @@ def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
                     f"({row['JobName']}) has low CPU efficiency: "
                     f"{row['CPU Efficiency (%)']}%."
                 )
+    return df
+
+
+def create_efficiency_report(e_threshold, run_uuid, e_report_path, logger):
+    """
+    Fetch sacct job data for a Snakemake workflow
+    and compute efficiency metrics.
+    """
+    lines = get_sacct_data(run_uuid, logger)
+
+    if lines is None or not lines:
+        return None
+
+    df = parse_sacct_data(lines, e_threshold, run_uuid, logger)
 
     # we construct a path object to allow for a customi
     # logdir, if specified

snakemake_executor_plugin_slurm/submit_string.py

@@ -55,15 +55,18 @@ def get_submit_command(job, params):
         # fixes #316 - allow unsetting of tasks per gpu
         # apparently, python's internal process manangement interfers with SLURM
         # e.g. for pytorch
-        ntasks_per_gpu = job.resources.get(
-            "tasks_per_gpu", job.resources.get("tasks", 1)
-        )
+        ntasks_per_gpu = job.resources.get("tasks_per_gpu")
+        if ntasks_per_gpu is None:
+            ntasks_per_gpu = job.resources.get("tasks")
+        if ntasks_per_gpu is None:
+            ntasks_per_gpu = 1
+
         if ntasks_per_gpu >= 1:
             call += f" --ntasks-per-gpu={ntasks_per_gpu}"
     else:
         # fixes #40 - set ntasks regardless of mpi, because
         # SLURM v22.05 will require it for all jobs
-        call += f" --ntasks={job.resources.get('tasks', 1)}"
+        call += f" --ntasks={job.resources.get('tasks') or 1}"
 
     # we need to set cpus-per-task OR cpus-per-gpu, the function
     # will return a string with the corresponding value

{snakemake_executor_plugin_slurm-1.6.0.dist-info → snakemake_executor_plugin_slurm-1.7.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: snakemake-executor-plugin-slurm
-Version: 1.6.0
+Version: 1.7.0
 Summary: A Snakemake executor plugin for submitting jobs to a SLURM cluster.
 License: MIT
 Keywords: snakemake,plugin,executor,cluster,slurm
@@ -15,8 +15,8 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: numpy (>=1.26.4,<3)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: snakemake-executor-plugin-slurm-jobstep (>=0.3.0,<0.4.0)
-Requires-Dist: snakemake-interface-common (>=1.13.0,<2.0.0)
-Requires-Dist: snakemake-interface-executor-plugins (>=9.1.1,<10.0.0)
+Requires-Dist: snakemake-interface-common (>=1.21.0,<2.0.0)
+Requires-Dist: snakemake-interface-executor-plugins (>=9.3.9,<10.0.0)
 Requires-Dist: throttler (>=1.2.2,<2.0.0)
 Project-URL: Documentation, https://snakemake.github.io/snakemake-plugin-catalog/plugins/executor/slurm.html
 Project-URL: Repository, https://github.com/snakemake/snakemake-executor-plugin-slurm

snakemake_executor_plugin_slurm-1.7.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+snakemake_executor_plugin_slurm/__init__.py,sha256=BOuvCB954jBaxqoh4ezFBYLStptsrESbPduDXMT0K9Q,33916
+snakemake_executor_plugin_slurm/efficiency_report.py,sha256=crPfJDK4NojfRbu_wEw3ZmC3suMRABr5r-1rO5q3WEo,7429
+snakemake_executor_plugin_slurm/submit_string.py,sha256=Cn9qopyQwBqs1MvZFxSyRR_7mZzCVj8_vO_JNzbiqew,2896
+snakemake_executor_plugin_slurm/utils.py,sha256=7XVXtzu7bg_89wWZisW-Zk7TNQyEgK4v_y4Y3F9uOwc,4491
+snakemake_executor_plugin_slurm-1.7.0.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
+snakemake_executor_plugin_slurm-1.7.0.dist-info/METADATA,sha256=XiUbU8KRQKmnK55EVkfrEKRYny3I4IF2WpbrfP43Acc,1434
+snakemake_executor_plugin_slurm-1.7.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+snakemake_executor_plugin_slurm-1.7.0.dist-info/RECORD,,

snakemake_executor_plugin_slurm-1.6.0.dist-info/RECORD

@@ -1,8 +0,0 @@
-snakemake_executor_plugin_slurm/__init__.py,sha256=VXA5Bn58OCd143Kb3STvv6YxTePUsE5wbfrKYXFIpvE,33978
-snakemake_executor_plugin_slurm/efficiency_report.py,sha256=Lcafd8Zv14NMOmghPzuUd7rxLUS8PQmOISuNosauA4w,6256
-snakemake_executor_plugin_slurm/submit_string.py,sha256=C4-ZIsrHJg9WiJuDW3HqLno0uigBz8IYvgbI6nldJ38,2789
-snakemake_executor_plugin_slurm/utils.py,sha256=7XVXtzu7bg_89wWZisW-Zk7TNQyEgK4v_y4Y3F9uOwc,4491
-snakemake_executor_plugin_slurm-1.6.0.dist-info/LICENSE,sha256=YVc4xTLWMqGfFL36120k7rzXtsT6e4RkJsh68VVn12s,1076
-snakemake_executor_plugin_slurm-1.6.0.dist-info/METADATA,sha256=qWKq4kHlkj-KFSdNk1YINAnL08M299SQTWBhqj566qo,1434
-snakemake_executor_plugin_slurm-1.6.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-snakemake_executor_plugin_slurm-1.6.0.dist-info/RECORD,,