stepup-queue 1.0.6__tar.gz → 1.1.0__tar.gz

{stepup_queue-1.0.6/stepup_queue.egg-info → stepup_queue-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup-queue
-Version: 1.0.6
+Version: 1.1.0
 Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later
@@ -24,7 +24,8 @@ Classifier: Topic :: Software Development :: Build Tools
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: stepup<4.0.0,>=3.1.3
+Requires-Dist: path>=16.14.0
+Requires-Dist: stepup<4.0.0,>=3.2.0
 Provides-Extra: dev
 Requires-Dist: psutil; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{stepup_queue-1.0.6 → stepup_queue-1.1.0}/pyproject.toml

@@ -28,7 +28,8 @@ classifiers = [
 ]
 dependencies = [
     # Ensure changes to these dependencies are reflected in .github/requirements-old.txt
-    "stepup>=3.1.3,<4.0.0",
+    "path>=16.14.0",
+    "stepup>=3.2.0,<4.0.0",
 ]
 dynamic = ["version"]
 
@@ -56,9 +57,10 @@ sbatch = "stepup.queue.actions:sbatch"
 
 [project.entry-points."stepup.tools"]
 canceljobs = "stepup.queue.canceljobs:canceljobs_subcommand"
+removejobs = "stepup.queue.removejobs:removejobs_subcommand"
 
 [tool.pytest.ini_options]
-addopts = "-n auto -W error -W ignore::ResourceWarning"
+addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
 asyncio_default_fixture_loop_scope = "function"
 
 [tool.ruff]

{stepup_queue-1.0.6 → stepup_queue-1.1.0}/stepup/queue/actions.py

@@ -28,7 +28,7 @@ from path import Path
 
 from stepup.core.worker import WorkThread
 
-from .canceljobs import read_jobid_cluster
+from .canceljobs import read_jobid_cluster_status
 from .sbatch import InpDigestError, submit_once_and_wait
 
 
@@ -48,7 +48,10 @@ def sbatch(argstr: str, work_thread: WorkThread) -> int:
             return submit_once_and_wait(work_thread, args.ext, args.rc)
         # Cancel running job (if any), clean log and resubmit
         path_log = Path("slurmjob.log")
-        job_id, cluster = read_jobid_cluster(path_log)
-        work_thread.runsh(f"scancel -M {cluster} {job_id}")
+        job_id, cluster, _ = read_jobid_cluster_status(path_log)
+        if cluster is None:
+            work_thread.runsh(f"scancel {job_id}")
+        else:
+            work_thread.runsh(f"scancel -M {cluster} {job_id}")
         path_log.remove_p()
     return submit_once_and_wait(work_thread, args.ext, args.rc, args.onchange != "ignore")

stepup_queue-1.1.0/stepup/queue/canceljobs.py

@@ -0,0 +1,117 @@
+# StepUp Queue integrates queued jobs into a StepUp workflow.
+# © 2025 Toon Verstraelen
+#
+# This file is part of StepUp Queue.
+#
+# StepUp Queue is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# StepUp Queue is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+#
+# --
+"""Tool to cancel jobs."""
+
+import argparse
+import subprocess
+import sys
+
+from path import Path
+
+from .sbatch import DONE_STATES, parse_sbatch, read_log, read_status
+from .utils import search_jobs
+
+
+def canceljobs_tool(args: argparse.Namespace):
+    """Iterate over all slurmjob.log files, read the SLURM job IDs, and cancel them."""
+    jobs = {}
+    for path_log in search_jobs(args.paths, verbose=True):
+        try:
+            job_id, cluster, status = read_jobid_cluster_status(path_log)
+        except ValueError as e:
+            print(f"# WARNING: Could not read job ID from {path_log}: {e}")
+            continue
+        if args.all or status not in DONE_STATES:
+            jobs.setdefault(cluster, []).append((job_id, path_log, status))
+
+    all_good = True
+    for cluster, cluster_jobs in jobs.items():
+        if args.commit:
+            # Cancel at most 100 at a time to avoid exceeding the command line length limit,
+            # and to play nice with SLURM.
+            while len(cluster_jobs) > 0:
+                cancel_jobs = cluster_jobs[:100]
+                cluster_jobs[:] = cluster_jobs[100:]
+
+                command_args = ["scancel"]
+                if cluster is not None:
+                    command_args.extend(["-M", cluster])
+                command_args.extend(str(job_id) for job_id, _, _ in cancel_jobs)
+
+                # Using subprocess.run for better control and error handling
+                print(" ".join(command_args))
+                result = subprocess.run(command_args, check=False)
+                all_good &= result.returncode == 0
+        else:
+            for job_id, path_log, status in cluster_jobs:
+                command = "scancel"
+                if cluster is not None:
+                    command += f" -M {cluster}"
+                command += f" {job_id}  # {path_log} {status}"
+                print(command)
+    if not all_good:
+        print("Some jobs could not be cancelled. See messages above.")
+        sys.exit(1)
+
+
+def read_jobid_cluster_status(path_log: str) -> tuple[int, str | None, str | None]:
+    """Read the job ID, cluster, and job status from the job log file."""
+    lines = read_log(path_log, False)
+    if len(lines) < 1:
+        raise ValueError(f"Incomplete file: {path_log}.")
+    words = lines[0].split()
+    if len(words) != 3:
+        raise ValueError(f"Could not read job ID from first status line: {lines[0]}")
+    _, status, job_id_cluster = words
+    if status != "Submitted":
+        raise ValueError(f"No 'Submitted' on first status line: {lines[0]}")
+    job_id, cluster = parse_sbatch(job_id_cluster)
+    status = read_status(lines[-1:])[1]
+    return job_id, cluster, status
+
+
+def canceljobs_subcommand(subparser: argparse.ArgumentParser) -> callable:
+    parser = subparser.add_parser(
+        "canceljobs",
+        help="Cancel running jobs in the current StepUp workflow.",
+    )
+    parser.add_argument(
+        "paths",
+        nargs="*",
+        default=[Path(".")],
+        type=Path,
+        help="Paths to the jobs to cancel. Subdirectories are searched recursively. "
+        "If not specified, the current directory is used.",
+    )
+    parser.add_argument(
+        "-c",
+        "--commit",
+        action="store_true",
+        default=False,
+        help="Execute the cancellation of jobs instead of only showing what would be done.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Select all jobs, including the ones that seem to be done already.",
+    )
+    return canceljobs_tool

stepup_queue-1.1.0/stepup/queue/removejobs.py

@@ -0,0 +1,99 @@
+# StepUp Queue integrates queued jobs into a StepUp workflow.
+# © 2025 Toon Verstraelen
+#
+# This file is part of StepUp Queue.
+#
+# StepUp Queue is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# StepUp Queue is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+#
+# --
+"""Tool to remove failed jobs."""
+
+import argparse
+import shutil
+
+from path import Path
+
+from .sbatch import read_log, read_status
+from .utils import search_jobs
+
+FAILED_STATES = {
+    "BOOT_FAIL",
+    "CANCELLED",
+    "DEADLINE",
+    "FAILED",
+    "NODE_FAIL",
+    "OUT_OF_MEMORY",
+    "PREEMPTED",
+    "TIMEOUT",
+    "LAUNCH_FAILED",
+    "RECONFIG_FAIL",
+    "REVOKED",
+    "STOPPED",
+}
+
+
+def removejobs_tool(args: argparse.Namespace):
+    """Iterate over all slurmjob.log files and remove their parent job directories."""
+    jobs = []
+    for path_log in search_jobs(args.paths, verbose=True):
+        try:
+            status = read_last_status(path_log)
+        except ValueError as e:
+            print(f"Warning: Could not read job status from {path_log}: {e}")
+            status = None
+        if args.all or status in FAILED_STATES:
+            jobs.append((path_log, status))
+
+    for path_log, status in jobs:
+        command = f"rm -rf {path_log.parent}  # state={status}"
+        print(command)
+        if args.commit:
+            shutil.rmtree(path_log.parent)
+
+
+def read_last_status(path_log: str) -> str | None:
+    """Read the last job status from the job log file."""
+    lines = read_log(path_log, False)
+    return read_status(lines[-1:])[1]
+
+
+def removejobs_subcommand(subparser: argparse.ArgumentParser) -> callable:
+    parser = subparser.add_parser(
+        "removejobs",
+        help="Remove directories of failed (and optionally all completed) jobs "
+        "in the current StepUp workflow.",
+    )
+    parser.add_argument(
+        "paths",
+        nargs="*",
+        default=[Path(".")],
+        type=Path,
+        help="Paths to the jobs to remove. Subdirectories are searched recursively. "
+        "If not specified, the current directory is used.",
+    )
+    parser.add_argument(
+        "-c",
+        "--commit",
+        action="store_true",
+        default=False,
+        help="Execute the removal of jobs instead of only showing what would be done.",
+    )
+    parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        default=False,
+        help="Remove all jobs, not only failed jobs.",
+    )
+    return removejobs_tool

{stepup_queue-1.0.6 → stepup_queue-1.1.0}/stepup/queue/sbatch.py

@@ -28,15 +28,17 @@ from datetime import datetime
 
 from path import Path
 
-from stepup.core.utils import string_to_bool
 from stepup.core.worker import WorkThread
 
 FIRST_LINE = "StepUp Queue sbatch wait log format version 2"
-SCONTROL_FAILED = "The command `scontrol show job` failed!\n"
-DEBUG = string_to_bool(os.getenv("STEPUP_SBATCH_DEBUG", "0"))
+SBATCH_RETRY_NUM = int(os.getenv("STEPUP_SBATCH_RETRY_NUM", "5"))
+SBATCH_RETRY_DELAY_MIN = int(os.getenv("STEPUP_SBATCH_RETRY_DELAY_MIN", "60"))
+SBATCH_RETRY_DELAY_MAX = int(os.getenv("STEPUP_SBATCH_RETRY_DELAY_MAX", "120"))
 CACHE_TIMEOUT = int(os.getenv("STEPUP_SBATCH_CACHE_TIMEOUT", "30"))
-POLLING_INTERVAL = int(os.getenv("STEPUP_SBATCH_POLLING_INTERVAL", "10"))
-TIME_MARGIN = int(os.getenv("STEPUP_SBATCH_TIME_MARGIN", "15"))
+POLLING_MIN = int(os.getenv("STEPUP_SBATCH_POLLING_MIN", "10"))
+POLLING_MAX = max(int(os.getenv("STEPUP_SBATCH_POLLING_MAX", "20")), POLLING_MIN)
+SACCT_START = os.getenv("STEPUP_SACCT_START_TIME", "now-7days")
+UNLISTED_TIMEOUT = int(os.getenv("STEPUP_SBATCH_UNLISTED_TIMEOUT", "600"))
 
 
 def submit_once_and_wait(
@@ -66,27 +68,26 @@ def submit_once_and_wait(
         The return code of the job.
         0 if successful, 1 if the job failed.
     """
-    # Read previously logged steps
+    # Read previously logged job states
     path_log = Path("slurmjob.log")
-    if path_log.is_file():
-        previous_lines = read_log(path_log, validate_inp_digest)
-    else:
-        previous_lines = []
-        _init_log(path_log)
+    previous_lines = read_log(path_log, validate_inp_digest) if path_log.is_file() else []
 
-    # Go through or skip steps.
-    submit_time, status = read_step(previous_lines)
+    # Go through or skip states.
+    submit_time, status = read_status(previous_lines)
     if status is None:
         # A new job must be submitted.
         submit_time = time.time()
         sbatch_stdout = submit_job(work_thread, job_ext, sbatch_rc)
-        log_step(path_log, f"Submitted {sbatch_stdout}")
+        # Create a new log file after submitting the job.
+        _init_log(path_log)
+        log_status(path_log, f"Submitted {sbatch_stdout}")
         rndsleep()
     else:
-        # The first step, if present in the log, is the submission.
-        step, sbatch_stdout = status.split()
-        if step != "Submitted":
-            raise ValueError(f"Expected 'Submitted' in log, found '{step}'")
+        # The first state, if present in the log, is the submission.
+        words = status.split()
+        if len(words) != 2 or words[0] != "Submitted":
+            raise ValueError(f"Expected 'Submitted' in log, found '{status}'")
+        sbatch_stdout = words[1]
     jobid, cluster = parse_sbatch(sbatch_stdout)
 
     # Wait for the job to complete
@@ -103,12 +104,17 @@ def submit_once_and_wait(
             work_thread, submit_time, jobid, cluster, previous_lines, path_log, status
         )
 
-    # Get the return code from the job
-    with open("slurmjob.ret") as fh:
-        returncode = fh.read().strip()
-    if returncode == "":
-        raise ValueError("The job did not return a return code, e.g. because it was cancelled.")
-    return int(returncode)
+    if status == "COMPLETED":
+        # Get the return code from the job
+        with open("slurmjob.ret") as fh:
+            returncode = fh.read().strip()
+        try:
+            return int(returncode)
+        except ValueError as exc:
+            raise ValueError(
+                f"Could not parse return code from slurmjob.ret. Got '{returncode}'"
+            ) from exc
+    raise RuntimeError(f"Job ended with status '{status}'.")
 
 
 def read_log(path_log: str, do_inp_digest: bool = True) -> list[str]:
@@ -122,7 +128,7 @@ def read_log(path_log: str, do_inp_digest: bool = True) -> list[str]:
         try:
             inp_digest = next(f).strip()
         except StopIteration as exc:
-            raise ValueError("Existing has no input digest.") from exc
+            raise ValueError("Existing log file has no input digest.") from exc
         if do_inp_digest:
             check_log_inp_digest(inp_digest)
         for line in f:
@@ -131,6 +137,14 @@ def read_log(path_log: str, do_inp_digest: bool = True) -> list[str]:
     return lines
 
 
+def check_log_version(line: str):
+    """Validate the log version, abort if there is a mismatch."""
+    if line != FIRST_LINE:
+        raise ValueError(
+            f"The first line of the log is wrong. Expected: '{FIRST_LINE}' Found: '{line}'"
+        )
+
+
 def _init_log(path_log: str):
     """Initialize a new log file."""
     inp_digest = os.getenv("STEPUP_STEP_INP_DIGEST")
@@ -141,6 +155,66 @@ def _init_log(path_log: str):
         print(inp_digest, file=fh)
 
 
+# From: https://slurm.schedmd.com/job_state_codes.html
+KNOWN_JOB_STATES = {
+    # -- Job states
+    # done
+    "BOOT_FAIL",
+    "CANCELLED",
+    "COMPLETED",
+    "DEADLINE",
+    "FAILED",
+    "NODE_FAIL",
+    "OUT_OF_MEMORY",
+    "PREEMPTED",
+    "TIMEOUT",
+    # waiting or running
+    "PENDING",
+    "RUNNING",
+    "SUSPENDED",
+    # -- Job flags
+    # done
+    "LAUNCH_FAILED",
+    "RECONFIG_FAIL",
+    "REVOKED",
+    "STOPPED",
+    # waiting or running
+    "COMPLETING",
+    "CONFIGURING",
+    "EXPEDITING",
+    "POWER_UP_NODE",
+    "REQUEUED",
+    "REQUEUE_FED",
+    "REQUEUE_HOLD",
+    "RESIZING",
+    "RESV_DEL_HOLD",
+    "SIGNALING",
+    "SPECIAL_EXIT",
+    "STAGE_OUT",
+    "UPDATE_DB",
+    # -- Specific to this script
+    # to be ignored (same as waiting or running), must not be logged
+    "invalid",
+    "unlisted",
+}
+
+DONE_STATES = {
+    "BOOT_FAIL",
+    "CANCELLED",
+    "COMPLETED",
+    "DEADLINE",
+    "FAILED",
+    "NODE_FAIL",
+    "OUT_OF_MEMORY",
+    "PREEMPTED",
+    "TIMEOUT",
+    "LAUNCH_FAILED",
+    "RECONFIG_FAIL",
+    "REVOKED",
+    "STOPPED",
+}
+
+
 def _read_or_poll_status(
     work_thread: WorkThread,
     submit_time: float,
@@ -155,7 +229,7 @@ def _read_or_poll_status(
     Parameters
     ----------
     work_thread
-        The work thread to use for launching the scontrol command.
+        The work thread to use for launching the sacct command.
     submit_time
         The timestamp when the job was submitted.
     jobid
@@ -165,7 +239,6 @@ def _read_or_poll_status(
     previous_lines
         Lines from an existing log file to be processed first.
         (It will be gradually emptied.)
-    path_log
         The log file to write new polling results to.
     last_status
         The status from the previous iteration.
@@ -178,29 +251,28 @@ def _read_or_poll_status(
     done
         True when the waiting is over.
     """
-    # First try to replay previously logged steps
-    status_time, status = read_step(previous_lines)
+    # First try to replay previously logged states
+    _, status = read_status(previous_lines)
     if status is None:
-        # All previously logged steps are processed.
-        # Call scontrol and parse its response.
+        # All previously logged states are processed.
+        # Call sacct and parse its response.
         rndsleep()
-        status_time, status = get_status(work_thread, jobid, cluster)
+        _, status = get_status(work_thread, jobid, cluster)
         # Log only if the status changed, and is not invalid or unlisted.
         # These two statuses are (potentially) transient and should not be logged.
         if status != last_status and status not in ["invalid", "unlisted"]:
-            log_step(path_log, status)
-    done = (status_time > submit_time + TIME_MARGIN) and (
-        status not in ["PENDING", "CONFIGURING", "RUNNING", "invalid"]
-    )
-    return status, done
+            log_status(path_log, status)
+    if status not in KNOWN_JOB_STATES:
+        raise ValueError(f"Unknown job status '{status}' obtained from scheduler.")
 
+    # Determine if the job is done
+    done = status in DONE_STATES
+    if status == "unlisted" and time.time() > submit_time + UNLISTED_TIMEOUT:
+        # If the job remains unlisted for too long, we declare it failed.
+        # This prevents an infinite loop if the job ID was wrong or purged.
+        done = True
 
-def check_log_version(line: str):
-    """Validate the log version, abort if there is a mismatch."""
-    if line != FIRST_LINE:
-        raise ValueError(
-            f"The first line of the log is wrong. Expected: '{FIRST_LINE}' Found: '{line}'"
-        )
+    return status, done
 
 
 class InpDigestError(ValueError):
@@ -219,20 +291,20 @@ def check_log_inp_digest(line: str):
         )
 
 
-def read_step(lines: list[str]) -> str | None:
-    """Read a step from the log file."""
+def read_status(lines: list[str]) -> tuple[float | None, str | None]:
+    """Read a status from the log file."""
     if len(lines) == 0:
         return None, None
     line = lines.pop(0)
     words = line.split(maxsplit=1)
     if len(words) != 2:
-        raise ValueError(f"Expected a step in log but found line '{line}'.")
-    return datetime.fromisoformat(words[0]).timestamp(), words[1]
+        raise ValueError(f"Expected a status in log but found line '{line}'.")
+    return datetime.fromisoformat(words[0]).timestamp(), words[1].strip()
 
 
 def rndsleep():
     """Randomized sleep to distribute I/O load evenly."""
-    sleep_seconds = 1 if DEBUG else random.randint(POLLING_INTERVAL, POLLING_INTERVAL + TIME_MARGIN)
+    sleep_seconds = random.randint(POLLING_MIN, POLLING_MAX)
     time.sleep(sleep_seconds)
 
 
@@ -241,43 +313,75 @@ JOB_SCRIPT_WRAPPER = """\
 {sbatch_header}
 
 touch slurmjob.ret
-chmod +x '{job_script}'
 ./'{job_script}'
 RETURN_CODE=$?
 echo $RETURN_CODE > slurmjob.ret
 exit $RETURN_CODE
 """
 
+RE_SBATCH_STDOUT = re.compile(r"\s*#\s*SBATCH\b.*(--output|-o)\b")
+RE_SBATCH_STDERR = re.compile(r"\s*#\s*SBATCH\b.*(--error|-e)\b")
+RE_SBATCH_ARRAY = re.compile(r"\s*#\s*SBATCH\b.*(--array|-a)\b")
+RE_SBATCH = re.compile(r"\s*#\s*SBATCH\b")
+UNSUPPORTED_DIRECTIVES = [
+    re.compile(r"\s*#\s*PBS\b"),
+    re.compile(r"\s*#\s*BSUB\b"),
+    re.compile(r"\s*#\s*COBALT\b"),
+    re.compile(r"\s*#\$"),
+]
+
 
 def submit_job(work_thread: WorkThread, job_ext: str, sbatch_rc: str | None = None) -> str:
     """Submit a job with sbatch."""
-    # Copy the #SBATCH lines from the job script.
+    # Verify that the job script is executable.
     path_job = f"slurmjob{job_ext}"
+    if not os.access(path_job, os.X_OK):
+        raise ValueError("The job script must be executable.")
+
+    # Copy the #SBATCH lines from the job script and perform some checks.
     with open(path_job) as f:
-        sbatch_header = "\n".join(line for line in f if line.startswith("#SBATCH"))
+        sbatch_header = []
+        first_line = next(f)
+        if not first_line.startswith("#!"):
+            raise ValueError("The job script must start with a shebang line.")
+        for line in f:
+            if RE_SBATCH_STDOUT.match(line):
+                raise ValueError("The job script must not contain a #SBATCH --output/-o line.")
+            if RE_SBATCH_STDERR.match(line):
+                raise ValueError("The job script must not contain a #SBATCH --error/-e line.")
+            if RE_SBATCH_ARRAY.match(line):
+                raise ValueError("StepUp Queue does not support array jobs. (Found -a or --array)")
+            if RE_SBATCH.match(line):
+                sbatch_header.append(line.strip())
+            else:
+                for pattern in UNSUPPORTED_DIRECTIVES:
+                    if pattern.match(line):
+                        raise ValueError(
+                            f"Detected unsupported scheduler directive: {line.strip()}."
+                        )
+        sbatch_header = "\n".join(sbatch_header)
 
     command = "sbatch --parsable -o slurmjob.out -e slurmjob.err"
     if sbatch_rc is not None:
         command = f"{sbatch_rc} < /dev/null && {command}"
-    returncode, stdout, stderr = work_thread.runsh(
-        command,
-        stdin=JOB_SCRIPT_WRAPPER.format(
-            sbatch_header=sbatch_header,
-            job_script=path_job,
-        ),
-    )
-    if returncode != 0:
+    stdin = JOB_SCRIPT_WRAPPER.format(sbatch_header=sbatch_header, job_script=path_job)
+    for _ in range(SBATCH_RETRY_NUM):
+        returncode, stdout, stderr = work_thread.runsh(command, stdin=stdin)
+        if returncode == 0:
+            return stdout.strip()
         if not (stderr is None or stderr == ""):
             print(stderr)
-        raise RuntimeError(f"sbatch failed with return code {returncode}.")
-    return stdout.strip()
+        delay = random.randint(SBATCH_RETRY_DELAY_MIN, SBATCH_RETRY_DELAY_MAX)
+        print(f"sbatch failed with return code {returncode}. Retrying in {delay} seconds.")
+        time.sleep(delay)
+    raise RuntimeError(f"sbatch failed {SBATCH_RETRY_NUM} times. Giving up.")
 
 
-def log_step(path_log: Path, step: str):
-    """Write a step to the log."""
+def log_status(path_log: Path, status: str):
+    """Write a status to the log."""
     dt = datetime.now().isoformat()
     with open(path_log, "a") as f:
-        line = f"{dt} {step}"
+        line = f"{dt} {status}"
         f.write(f"{line}\n")
 
 
@@ -291,13 +395,13 @@ def parse_sbatch(stdout: str) -> tuple[int, str | None]:
     raise ValueError(f"Cannot parse sbatch output: {stdout}")
 
 
-def get_status(work_thread: WorkThread, jobid: int, cluster: str | None) -> str:
-    """Load cached scontrol output or run scontrol if outdated.
+def get_status(work_thread: WorkThread, jobid: int, cluster: str | None) -> tuple[float, str]:
+    """Load cached sacct output or run sacct if outdated.
 
     Parameters
     ----------
     work_thread
-        The work thread to use for launching the scontrol command.
+        The work thread to use for launching the sacct command.
     jobid
         The job to wait for.
     cluster
@@ -305,25 +409,25 @@ def get_status(work_thread: WorkThread, jobid: int, cluster: str | None) -> str:
 
     Returns
     -------
+    timestamp
+        The time when the status was last retrieved.
     status
-        A status reported by scontrol,
-        or `invalid` if scontrol failed (retry scontrol later),
+        A status reported by sacct,
+        or `invalid` if sacct failed (retry sacct later),
         or `unlisted` if the job is not found (probably ended long ago).
     """
     # Load cached output or run again
-    command = "scontrol show job"
-    path_out = Path(os.getenv("HOME")) / ".cache/stepup-queue"
+    command = f"sacct -o 'jobid,state' -PXn -S {SACCT_START}"
+    path_out = Path(os.getenv("ROOT", ".")) / ".stepup/queue"
     if cluster is None:
-        path_out /= "sbatch_wait.out"
+        path_out /= "sbatch_wait_sacct.out"
     else:
         command += f" --cluster={cluster}"
-        path_out /= f"sbatch_wait.{cluster}.out"
-    status_time, scontrol_out, returncode = cached_run(
-        work_thread, command, path_out, CACHE_TIMEOUT
-    )
+        path_out /= f"sbatch_wait_sacct.{cluster}.out"
+    status_time, sacct_out, returncode = cached_run(work_thread, command, path_out, CACHE_TIMEOUT)
     if returncode != 0:
         return status_time, "invalid"
-    return status_time, parse_scontrol_out(scontrol_out, jobid)
+    return status_time, parse_sacct_out(sacct_out, jobid)
 
 
 def cached_run(
@@ -385,11 +489,14 @@ def make_cache_header(cache_time: float, returncode: int):
     """Prepare a header for the file containing the cached output of a cached execution."""
     iso = datetime.fromtimestamp(cache_time).isoformat()
     if len(iso) != 26:
-        raise AssertionError
-    return f"v1 datetime={iso} returncode={returncode:+04d}\n"
+        raise RuntimeError("ISO datetime string has unexpected length.")
+    returnstr = f"{returncode:+04d}"
+    if len(returnstr) != 4:
+        raise RuntimeError("Return code string has unexpected length.")
+    return f"v1 datetime={iso} returncode={returnstr}\n"
 
 
-def parse_cache_header(header: str) -> tuple[float, int]:
+def parse_cache_header(header: str) -> tuple[float, int] | tuple[None, None]:
     """Read the header of a cached output and return the timestamp and returncode."""
     if len(header) == 0 or header == "\x00" * CACHE_HEADER_LENGTH:
         return None, None
@@ -405,30 +512,31 @@ def parse_cache_header(header: str) -> tuple[float, int]:
 CACHE_HEADER_LENGTH = len(make_cache_header(time.time(), 0))
 
 
-def parse_scontrol_out(scontrol_out: str, jobid: int) -> str:
-    """Get the job state for a specific from from the output of ``scontrol show job``.
+def parse_sacct_out(sacct_out: str, jobid: int) -> str:
+    """Get the job state for a specific from from the output of ``sacct -o 'jobid,state' -PXn``.
 
     Parameters
     ----------
-    scontrol_out
-        A string with the output of ``scontrol show job``.
+    sacct_out
+        A string with the output of ``sacct -o 'jobid,state' -PXn``.
     jobid
         The jobid of interest.
 
     Returns
     -------
-    jobstate
+    status
         The status of the job. This can be:
 
         - Any of the SLURM job states.
         - `unlisted` if the job cannot be found,
           which practically means it has ended long ago.
+        - `invalid` if the sacct output cannot be parsed.
     """
-    match = re.search(
-        f"JobId={jobid}.*?JobState=(?P<state>[A-Z]+)",
-        scontrol_out,
-        flags=re.MULTILINE | re.DOTALL,
-    )
-    if match is not None:
-        return match.group("state")
+    try:
+        for line in sacct_out.splitlines():
+            columns = line.strip().split("|")
+            if int(columns[0]) == jobid:
+                return columns[1].strip().split()[0]
+    except (ValueError, IndexError):
+        return "invalid"
     return "unlisted"

stepup_queue-1.1.0/stepup/queue/utils.py

@@ -0,0 +1,58 @@
+# StepUp Queue integrates queued jobs into a StepUp workflow.
+# © 2025 Toon Verstraelen
+#
+# This file is part of StepUp Queue.
+#
+# StepUp Queue is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# StepUp Queue is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+#
+# --
+"""Utility functions for the StepUp queue module."""
+
+from itertools import chain
+
+from path import Path
+
+__all__ = ("search_jobs",)
+
+
+def search_jobs(paths: list[Path], verbose: bool = False) -> list[Path]:
+    """Recursively search for slurmjob.log files in the specified directories.
+
+    Parameters
+    ----------
+    paths
+        List of directories to search in.
+    verbose
+        Whether to print warnings when paths do not exist or are not directories.
+
+    Returns
+    -------
+    paths_log
+        Sorted list of found slurmjob.log file paths.
+    """
+    paths_log = set()
+    for path in paths:
+        if not path.exists():
+            if verbose:
+                print(f"# WARNING: Path {path} does not exist.")
+            continue
+        if not path.is_dir():
+            if verbose:
+                print(f"# WARNING: Path {path} is not a directory.")
+            continue
+        for path_sub in chain([path], path.walkdirs()):
+            path_log = path_sub / "slurmjob.log"
+            if path_log.is_file():
+                paths_log.add(path_log)
+    return sorted(paths_log)

{stepup_queue-1.0.6 → stepup_queue-1.1.0/stepup_queue.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup-queue
-Version: 1.0.6
+Version: 1.1.0
 Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later
@@ -24,7 +24,8 @@ Classifier: Topic :: Software Development :: Build Tools
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: stepup<4.0.0,>=3.1.3
+Requires-Dist: path>=16.14.0
+Requires-Dist: stepup<4.0.0,>=3.2.0
 Provides-Extra: dev
 Requires-Dist: psutil; extra == "dev"
 Requires-Dist: pytest; extra == "dev"

{stepup_queue-1.0.6 → stepup_queue-1.1.0}/stepup_queue.egg-info/SOURCES.txt

@@ -6,7 +6,9 @@ stepup/queue/__init__.py
 stepup/queue/actions.py
 stepup/queue/api.py
 stepup/queue/canceljobs.py
+stepup/queue/removejobs.py
 stepup/queue/sbatch.py
+stepup/queue/utils.py
 stepup_queue.egg-info/PKG-INFO
 stepup_queue.egg-info/SOURCES.txt
 stepup_queue.egg-info/dependency_links.txt

{stepup_queue-1.0.6 → stepup_queue-1.1.0}/stepup_queue.egg-info/entry_points.txt

@@ -3,3 +3,4 @@ sbatch = stepup.queue.actions:sbatch
 
 [stepup.tools]
 canceljobs = stepup.queue.canceljobs:canceljobs_subcommand
+removejobs = stepup.queue.removejobs:removejobs_subcommand

{stepup_queue-1.0.6 → stepup_queue-1.1.0}/stepup_queue.egg-info/requires.txt

@@ -1,4 +1,5 @@
-stepup<4.0.0,>=3.1.3
+path>=16.14.0
+stepup<4.0.0,>=3.2.0
 
 [dev]
 psutil

stepup_queue-1.0.6/stepup/queue/canceljobs.py

@@ -1,77 +0,0 @@
-# StepUp Queue integrates queued jobs into a StepUp workflow.
-# © 2025 Toon Verstraelen
-#
-# This file is part of StepUp Queue.
-#
-# StepUp Queue is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# StepUp Queue is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>
-#
-# --
-"""Tool to cancel jobs."""
-
-import argparse
-import os
-
-from path import Path
-
-from .sbatch import FIRST_LINE
-
-
-def canceljobs_tool(args: argparse.Namespace) -> int:
-    if len(args.paths) == 0:
-        args.paths = [Path(".")]
-    # Iterate over all slurmjob.log files in the specified directories, and kill them.
-    job_ids = {}
-    for path in args.paths:
-        if not path.exists():
-            print(f"Path {path} does not exist.")
-            continue
-        if not path.is_dir():
-            print(f"Path {path} is not a directory.")
-            continue
-        for job_log in path.glob("**/slurmjob.log"):
-            job_id, cluster = read_jobid_cluster(job_log)
-            print(f"Found job {job_id} on cluster {cluster} in {job_log}")
-            job_ids.setdefault(cluster, []).append(job_id)
-    # Cancel 100 at a time to avoid exceeding the command line length limit.
-    for cluster, cluster_job_ids in job_ids.items():
-        while len(cluster_job_ids) > 0:
-            command = f"scancel -M {cluster} " + " ".join(cluster_job_ids[:100])
-            print(command)
-            os.system(command)
-            cluster_job_ids[:] = cluster_job_ids[100:]
-
-
-def read_jobid_cluster(job_log: Path) -> tuple[str, str]:
-    """Read the job ID and cluster from the job log file."""
-    with open(job_log) as f:
-        lines = f.readlines()
-        if len(lines) < 3 or lines[0][:-1] != FIRST_LINE:
-            raise ValueError(f"Invalid first line in {job_log}.")
-        job_id, cluster = lines[2].split()[-1].split(";")
-    return job_id, cluster
-
-
-def canceljobs_subcommand(subparser: argparse.ArgumentParser) -> callable:
-    parser = subparser.add_parser(
-        "canceljobs",
-        help="Cancel running jobs in the current StepUp workflow.",
-    )
-    parser.add_argument(
-        "paths",
-        nargs="*",
-        type=Path,
-        help="Paths to the jobs to cancel. Subdirectories are searched recursively. "
-        "If not specified, the current directory is used.",
-    )
-    return canceljobs_tool