stepup-queue 1.0.1__tar.gz → 1.0.3__tar.gz

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/.pre-commit-config.yaml

@@ -28,6 +28,7 @@ repos:
   rev: v0.8.4
   hooks:
     - id: ruff-format
+      exclude: "^docs/examples/slurm-basic/pass/slurmjob.py$"
     - id: ruff
       args: ["--fix", "--show-fixes"]
 - repo: https://github.com/DavidAnson/markdownlint-cli2

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup-queue
-Version: 1.0.1
+Version: 1.0.3
 Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later
@@ -47,8 +47,9 @@ Dynamic: license-file
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
 ![GPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-queue)
 
-StepUp Queue is an experimental extension of
-[StepUp Core](https://reproducible-reporting.github.io/stepup-core)
-to integrate queued jobs into a workflow.
+StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
+extension to integrate queued jobs into a workflow.
 Currently, it only supports integration with [SLURM](https://slurm.schedmd.com/),
 but it is designed to be extensible to other job schedulers.
+
+For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/README.md

@@ -9,8 +9,9 @@
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
 ![GPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-queue)
 
-StepUp Queue is an experimental extension of
-[StepUp Core](https://reproducible-reporting.github.io/stepup-core)
-to integrate queued jobs into a workflow.
+StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
+extension to integrate queued jobs into a workflow.
 Currently, it only supports integration with [SLURM](https://slurm.schedmd.com/),
 but it is designed to be extensible to other job schedulers.
+
+For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/docs/changelog.md

@@ -12,6 +12,25 @@ and this project adheres to [Effort-based Versioning](https://jacobtomlinson.dev
 
 (no changes yet)
 
+## [1.0.3][] - 2025-05-16 {: #v1.0.3 }
+
+### Fixed
+
+- Fixed errors in the example job scripts.
+- Improved handling of `scontrol` failures.
+
+### Added
+
+## [1.0.2][] - 2025-05-14 {: #v1.0.2 }
+
+### Added
+
+- Option to specify the extension of the job script.
+- Wrap all job scripts to record their return code.
+- Detect when inputs of jobs have changed + optional resubmission.
+- Option to load resource configurations before sbatch is called.
+- More detailed examples, including a self-submitting workflow job.
+
 ## [1.0.1][] - 2025-05-11 {: #v1.0.1 }
 
 This is a minor cleanup release, mainly testing the release process.
@@ -28,5 +47,7 @@ It was adapted to integrate well with StepUp Core 3.
 This release also features the `stepup canceljobs` tool, which was not present in Parman.
 
 [Unreleased]: https://github.com/reproducible-reporting/stepup-queue
+[1.0.3]: https://github.com/reproducible-reporting/stepup-queue/releases/tag/v1.0.3
+[1.0.2]: https://github.com/reproducible-reporting/stepup-queue/releases/tag/v1.0.2
 [1.0.1]: https://github.com/reproducible-reporting/stepup-queue/releases/tag/v1.0.1
 [1.0.0]: https://github.com/reproducible-reporting/stepup-queue/releases/tag/v1.0.0

{stepup_queue-1.0.1/docs/examples/slurm → stepup_queue-1.0.3/docs/examples/slurm-basic}/.gitignore

@@ -3,3 +3,4 @@ dynamic?
 *.log
 *.out
 *.err
+*.ret

stepup_queue-1.0.3/docs/examples/slurm-basic/README.md

@@ -0,0 +1,50 @@
+# Basic SLURM example
+
+The latest version of this example can be found at:
+<https://github.com/reproducible-reporting/stepup-queue/tree/main/docs/examples/slurm-basic/>
+
+This example shows how to use StepUp to run job scripts,
+which can be either manually written (static) or generated from a template (dynamic).
+Since these jobs only take a few seconds and don't perform any computations,
+they allow for a quick demonstration of StepUp Queue's features.
+
+## Files
+
+```text
+.
+├── dynamic-template.sh
+├── fail
+│   └── slurmjob.sh
+├── pass
+│   └── slurmjob.py
+├── plan.py
+└── README.md
+```
+
+`plan.py` is a Python script that defines the workflow:
+
+```python
+{% include 'examples/slurm-basic/plan.py' %}
+```
+
+The job `fail/slurmjob.sh` is a static job script that fails with a non-zero exit code,
+which is correctly handled by StepUp Queue:
+
+```bash
+{% include 'examples/slurm-basic/fail/slurmjob.sh' %}
+```
+
+The job `pass/slurmjob.py` shows how to write a Job script in Python:
+
+```python
+{% include 'examples/slurm-basic/pass/slurmjob.py' %}
+```
+
+The file `dynamic-template.sh` is a template from which actual job scripts are generated:
+
+```bash
+{% include 'examples/slurm-basic/dynamic-template.sh' %}
+```
+
+Note that `render_jinja` can be used to render any kind of text-based file from a template,
+such as inputs to computational tools, configuration files, etc.

{stepup_queue-1.0.1/docs/examples/slurm → stepup_queue-1.0.3/docs/examples/slurm-basic}/dynamic-template.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
-#SBATCH -J 'dynamic {{ field }}'
-#SBATCH -N 1
+#SBATCH --job-name 'dyn{{ field }}'
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
 
 echo "Hello from dynamic job {{ field }}"
 sleep 5

stepup_queue-1.0.3/docs/examples/slurm-basic/fail/slurmjob.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+#SBATCH --job-name fail
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+
+echo "This job will fail"
+exit 1

stepup_queue-1.0.3/docs/examples/slurm-basic/pass/slurmjob.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+#SBATCH --job-name pass
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+
+from time import sleep
+
+print("Hello from static job")
+sleep(5)
+print("Goodbye from static job")

stepup_queue-1.0.3/docs/examples/slurm-basic/plan.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+from stepup.core.api import mkdir, render_jinja, static
+from stepup.queue.api import sbatch
+
+# Two examples of a static job script, i.e. already present on disk.
+static("pass/", "pass/slurmjob.py")
+sbatch("pass", ext=".py")
+static("fail/", "fail/slurmjob.sh")
+sbatch("fail")
+
+# Example of job scripts generated from a template.
+static("dynamic-template.sh")
+for i in range(1, 4):
+    mkdir(f"dynamic{i}/")
+    render_jinja("dynamic-template.sh", {"field": i}, f"dynamic{i}/slurmjob.sh")
+    # You can use the rc option to load an environment before calling sbatch.
+    # Use this only if it cannot be done in the job script itself.
+    sbatch(f"dynamic{i}/", rc="module swap cluster/doduo")

stepup_queue-1.0.3/docs/examples/slurm-perpetual/.gitignore

@@ -0,0 +1,6 @@
+.stepup
+intermediate.txt
+*.log
+*.out
+*.err
+*.ret

stepup_queue-1.0.3/docs/examples/slurm-perpetual/README.md

@@ -0,0 +1,58 @@
+# Perpetual SLURM Workflow Job
+
+The latest version of this example can be found at:
+<https://github.com/reproducible-reporting/stepup-queue/tree/main/docs/examples/slurm-perpetual/>
+
+For extensive workflows, it is often useful to submit the workflow itself to the queue as a job.
+It is generally preferred to run the workflow on a compute node of the cluster,
+as this allows for better resource management and prevents overloading the login node.
+However, most clusters impose a limit on the maximum wall time of a job,
+which can result in the workflow job being interrupted.
+This example shows how to work around this limitation by using a perpetual self-submitting job.
+
+At the start of the job, a background process is launched that will end StepUp
+before the wall time limit is reached if StepUp has not ended on its own.
+When StepUp is interrupted, a temporary file is created.
+This file is later used as a signal that the workflow job needs to be resubmitted.
+This technique can be used with any type of job and is not specific to StepUp.
+
+Here, we use a very short runtime to quickly demonstrate StepUp Queue's features.
+In practice, you can let the StepUp job run for several hours or even days at a time,
+and stop it about 30 minutes before the wall time limit is reached.
+
+## Files
+
+```text
+.
+├── plan.py
+├── README.md
+├── step1
+│   └── slurmjob.sh
+├── step2
+│   └── slurmjob.sh
+└── workflow.sh
+```
+
+`plan.py` is a Python script that defines the workflow:
+
+```python
+{% include 'examples/slurm-perpetual/plan.py' %}
+```
+
+`step1/slurmjob.sh` is the first SLURM job:
+
+```bash
+{% include 'examples/slurm-perpetual/step1/slurmjob.sh' %}
+```
+
+`step2/slurmjob.sh` is the second SLURM job:
+
+```bash
+{% include 'examples/slurm-perpetual/step2/slurmjob.sh' %}
+```
+
+`workflow.sh` is the SLURM job script that runs the workflow:
+
+```bash
+{% include 'examples/slurm-perpetual/workflow.sh' %}
+```

stepup_queue-1.0.3/docs/examples/slurm-perpetual/plan.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+
+from stepup.core.api import static
+from stepup.queue.api import sbatch
+
+static("step1/", "step1/slurmjob.sh", "step2/", "step2/slurmjob.sh")
+sbatch("step1/", out="../intermediate.txt")
+sbatch("step2/", inp="../intermediate.txt")

stepup_queue-1.0.3/docs/examples/slurm-perpetual/step1/slurmjob.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#SBATCH --job-name step1
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --time=00:02:00
+
+# Give the CPU a break...
+sleep 30
+echo Done > ../intermediate.txt

stepup_queue-1.0.3/docs/examples/slurm-perpetual/step2/slurmjob.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+#SBATCH --job-name step2
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+
+#SBATCH --time=00:02:00
+
+# Give the CPU a break...
+sleep 30
+cat ../intermediate.txt

stepup_queue-1.0.3/docs/examples/slurm-perpetual/workflow.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#SBATCH --job-name stepup
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=1
+#SBATCH --output=stepup-%j.out
+#SBATCH --time=00:01:00
+
+# In production, --time=12:00:00 is a reasonable time limit.
+echo "StepUp workflow job starts:" $(date)
+
+# If needed, load required modules and activate a relevant virtual environment.
+# For example:
+# module load Python/3.12.3
+# activate venv/bin/activate
+
+# Create a temporary directory to store a file that will be used as a flag
+# to indicate that resubmission is needed.
+STEPUP_QUEUE_FLAG_DIR=$(mktemp -d)
+echo "Created temporary directory: $STEPUP_QUEUE_FLAG_DIR"
+trap 'rm -rv "$STEPUP_QUEUE_FLAG_DIR"' EXIT
+
+# Start a background process that will end stepup near the wall time limit.
+# The first shutdown will wait for running steps to completed.
+# The second will forcefully terminate remaining running steps.
+echo "Starting background process to monitor wall time."
+(
+    sleep 30;  # In production, 39600 seconds is reasonable.
+    touch ${STEPUP_QUEUE_FLAG_DIR}/resubmit;
+    stepup shutdown;
+    sleep 10;  # In production, 300 seconds is reasonable.
+    stepup shutdown
+) &
+BGPID=$!
+trap "kill $BGPID" EXIT
+
+# Start StepUp with 5 workers.
+# This means that at most 5 jobs will be submitted concurrently.
+# You can adjust the number of workers based on your needs.
+# In fact, because this example is simple, a single worker would be sufficient.
+# Note that the number of workers is unrelated
+# to the single core used by this workflow script.
+echo "Starting stepup with a maximum of 5 concurrent jobs."
+stepup boot -n 5
+
+# Use the temporary file to determine if the workflow script must be resubmitted.
+echo "Checking if stepup was forcibly stopped."
+if [ -f ${STEPUP_QUEUE_FLAG_DIR}/resubmit ]; then
+    echo "Resubmitting job script to let StepUp finalize the workflow."
+    sbatch workflow.sh
+else
+    echo "Stepup was stopped gracefully."
+fi
+
+echo "StepUp workflow job ends:" $(date)

stepup_queue-1.0.3/docs/stepup.queue.api.md

@@ -0,0 +1,6 @@
+# stepup.queue.api
+
+::: stepup.queue.api
+      options:
+        docstring_style: numpy
+        show_root_heading: false

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/docs/usage.md

@@ -5,6 +5,7 @@
 If you want to submit a job to the queue as part of a StepUp workflow,
 you must first prepare a directory with a job script called `slurmjob.sh`.
 This can be either a static file or the output of a previous step in the workflow.
+The function [`sbatch()`][stepup.queue.api.sbatch] will then submit the job to the queue.
 For simplicity, the following example assumes that the job script is static:
 
 ```python
@@ -15,7 +16,8 @@ static("compute/", "compute/slurmjob.sh")
 sbatch("compute/")
 ```
 
-All arguments to `sbatch` must be included in the `slurmjob.sh` script with `#SBATCH` directives.
+All arguments to the `sbatch` command of SLURM
+must be included in the `slurmjob.sh` script with `#SBATCH` directives.
 You can only submit one job from a given directory.
 
 When the workflow is executed, the `sbatch` step will submit the job to the queue.
@@ -26,20 +28,29 @@ This can be useful when the workflow gets killed for some reason.
 The standard output and error of the job are written to `slurmjob.out` and `slurmjob.err`, respectively.
 
 The current status of the job is written to (and read from) the `slurmjob.log` file.
-The job will not be resubmitted if `slurmjob.log` exists.
-Instead, it will wait for the job to complete without resubmitting it.
+By default, the job is not resubmitted if `slurmjob.log` exists.
+Instead, it waits for the job to complete without resubmitting it.
 You can remove `slurmjob.log` to ensure that the job is resubmitted,
-but this is off course dangerous if the job is still running.
+but this is obviously dangerous if the job is still running.
 
-## Simple Example
+If the inputs of the job specified with `sbatch("compute/", inp=["inp.txt"])` have changed,
+restarting the workflow will by default raise an exception.
+Ideally, you should clean up old outputs before restarting the workflow,
+and check that you really want to remove the data before doing so.
+If you feel this is overly cautious, you can set the `STEPUP_QUEUE_RESUBMIT_CHANGED_INPUTS`
+environment variable to `"yes"` to allow the workflow to resubmit jobs with changed inputs.
+Old outputs are not removed before resubmission.
+It is assumed that your job script will perform the necessary cleanup itself.
 
-A simple working example with static and dynamically generated job scripts
-can be found in the [`examples/slurm/`](https://github.com/reproducible-reporting/stepup-queue/tree/main/docs/examples/slurm/)
-directory.
+## Examples
 
-```python
-{% include 'examples/slurm/plan.py' %}
-```
+- A simple example with static and dynamically generated job scripts
+  can be found in the [`examples/slurm-basic/`](examples/slurm-basic/README.md).
+
+- The example [`examples/slurm-perpetual/`](examples/slurm-perpetual/README.md)
+  shows how to run StepUp itself as a job in the queue,
+  which cancels and submits itself again when nearing the wall time limit,
+  if the workflow has not yet completed.
 
 ## Killing running jobs
 
@@ -53,9 +64,9 @@ stepup canceljobs
 ```
 
 It is part of the design of StepUp Queue's not to automatically cancel jobs when the workflow is interrupted.
-It is quite common for a workflow to be interrupted by accident or due to a technical problem.
+It is quite common for a workflow to be interrupted by accident or for technical reasons.
 In this case, it would be inefficient to also cancel running jobs, which may still be doing useful work.
-Instead, they continue to run and you can restart the StepUp workflow to pick up where it left off.
+Instead, jobs continue to run and you can restart the StepUp workflow to pick up where it left off.
 
 After having cancelled jobs, it is still your responsibility to clean up files in the workflow.
 Removing them is not always desirable, so this is not done automatically.

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/mkdocs.yaml

@@ -25,6 +25,8 @@ extra:
     provider: mike
     alias: true
     default: stable
+  # Workaround for showing an example with a Jinja2 placeholder
+  field: "{{ field }}"
 
 theme:
   name: material
@@ -64,6 +66,9 @@ nav:
   - Home: index.md
   - installation.md
   - usage.md
+  - examples/slurm-basic/README.md
+  - examples/slurm-perpetual/README.md
+  - stepup.queue.api.md
   - changelog.md
   - development.md
   - license.md

stepup_queue-1.0.3/stepup/queue/actions.py

@@ -0,0 +1,51 @@
+# StepUp Queue integrates queued jobs into a StepUp workflow.
+# © 2025 Toon Verstraelen
+#
+# This file is part of StepUp Queue.
+#
+# StepUp Queue is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# StepUp Queue is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, see <http://www.gnu.org/licenses/>
+#
+# --
+"""StepUp Queue package."""
+
+import argparse
+import contextlib
+import os
+import shlex
+
+from path import Path
+
+from stepup.core.utils import string_to_bool
+from stepup.core.worker import WorkThread
+
+from .canceljobs import read_jobid_cluster
+from .sbatch import InpDigestError, submit_once_and_wait
+
+
+def sbatch(argstr: str, work_thread: WorkThread) -> int:
+    # Use argparse to parse the argstr
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ext", nargs="?", default=".sh")
+    parser.add_argument("--rc", default=None)
+    args = parser.parse_args(shlex.split(argstr))
+
+    if string_to_bool(os.getenv("STEPUP_QUEUE_RESUBMIT_CHANGED_INPUTS", "0")):
+        with contextlib.suppress(InpDigestError):
+            return submit_once_and_wait(work_thread, args.ext, args.rc)
+        # Cancel running job (if any), clean log and resubmit
+        path_log = Path("slurmjob.log")
+        job_id, cluster = read_jobid_cluster(path_log)
+        work_thread.runsh(f"scancel -M {cluster} {job_id}")
+        path_log.remove_p()
+    return submit_once_and_wait(work_thread, args.ext, args.rc)

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/stepup/queue/api.py

@@ -19,15 +19,20 @@
 # --
 """StepUp Queue API functions to build workflows."""
 
+import shlex
 from collections.abc import Collection
 
 from stepup.core.api import step
 from stepup.core.utils import string_to_list
 
+__all__ = ("sbatch",)
+
 
 def sbatch(
     workdir: str,
     *,
+    ext: str = ".sh",
+    rc: str | None = None,
     inp: Collection[str] | str = (),
     env: Collection[str] | str = (),
     out: Collection[str] | str = (),
@@ -40,10 +45,11 @@ def sbatch(
 
     The following filename conventions are used in the given working directory:
 
-    - `job.sh` is the job script to be submitted.
-    - `job.log` is StepUp Queue's log file keeping track of the job's status.
-    - `job.out` is the job's output file (written by SLURM).
-    - `job.err` is the job's error file (written by SLURM).
+    - `slurmjob{ext}` is the job script to be submitted.
+    - `slurmjob.log` is StepUp Queue's log file keeping track of the job's status.
+    - `slurmjob.out` is the job's output file (written by SLURM).
+    - `slurmjob.err` is the job's error file (written by SLURM).
+    - `slurmjob.ret` is the job's return code (written by a wrapper script).
 
     Hence, you can only have one job script per working directory,
     and it is strongly recommended to use meaningful directory names.
@@ -55,12 +61,38 @@ def sbatch(
 
     See `step()` documentation in StepUp Core for all optional arguments.
     and the return value.
+
+    Parameters
+    ----------
+    ext
+        The filename extension of the jobscript.
+        The full name is `f"slurmjob{ext}"`.
+        Extensions `.log`, `.out`, `.err` and `.ret` are not allowed.
+    rc
+        A resource configuration to be executed before calling sbatch.
+        This will be executed in the same shell, right before the sbatch command.
+        For example, you can run `module swap cluster/something`
+        or prepare other resources.
+        If multiple instructions are needed, put them in a file, e.g. `rc.sh`
+        and pass it here as `source rc.sh`.
+        In this case, you usually also want to include `rc.sh` in the `inp` list.
     """
+    if ext == "":
+        ext = ".sh"
+    elif ext[0] != ".":
+        ext = f".{ext}"
+    if ext in [".log", ".out", ".err", ".ret"]:
+        raise ValueError(f"Invalid extension {ext}. The extension must not be .log, .out or .err.")
+    action = "sbatch"
+    if ext != ".sh":
+        action += f" {ext}"
+    if rc is not None:
+        action += f" --rc={shlex.quote(rc)}"
     return step(
-        "sbatch",
-        inp=["slurmjob.sh", *string_to_list(inp)],
+        action,
+        inp=[f"slurmjob{ext}", *string_to_list(inp)],
         env=env,
-        out=["slurmjob.out", "slurmjob.err", *string_to_list(out)],
+        out=["slurmjob.out", "slurmjob.err", "slurmjob.ret", *string_to_list(out)],
         vol=["slurmjob.log", *string_to_list(vol)],
         workdir=workdir,
         optional=optional,

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/stepup/queue/canceljobs.py

@@ -40,14 +40,9 @@ def canceljobs_tool(args: argparse.Namespace) -> int:
             print(f"Path {path} is not a directory.")
             continue
         for job_log in path.glob("**/slurmjob.log"):
-            with open(job_log) as f:
-                lines = f.readlines()
-                if len(lines) < 2 or lines[0][:-1] != FIRST_LINE:
-                    print(f"Invalid first line in {job_log}.")
-                    continue
-                job_id, cluster = lines[1].split()[-1].split(";")
-                print(f"Found job {job_id} on cluster {cluster} in {job_log}")
-                job_ids.setdefault(cluster, []).append(job_id)
+            job_id, cluster = read_jobid_cluster(job_log)
+            print(f"Found job {job_id} on cluster {cluster} in {job_log}")
+            job_ids.setdefault(cluster, []).append(job_id)
     # Cancel 100 at a time to avoid exceeding the command line length limit.
     for cluster, cluster_job_ids in job_ids.items():
         while len(cluster_job_ids) > 0:
@@ -57,6 +52,16 @@ def canceljobs_tool(args: argparse.Namespace) -> int:
             cluster_job_ids[:] = cluster_job_ids[100:]
 
 
+def read_jobid_cluster(job_log: Path) -> tuple[str, str]:
+    """Read the job ID and cluster from the job log file."""
+    with open(job_log) as f:
+        lines = f.readlines()
+        if len(lines) < 3 or lines[0][:-1] != FIRST_LINE:
+            raise ValueError(f"Invalid first line in {job_log}.")
+        job_id, cluster = lines[2].split()[-1].split(";")
+    return job_id, cluster
+
+
 def canceljobs_subcommand(subparser: argparse.ArgumentParser) -> callable:
     parser = subparser.add_parser(
         "canceljobs",

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/stepup/queue/sbatch.py

@@ -31,7 +31,7 @@ from path import Path
 from stepup.core.utils import string_to_bool
 from stepup.core.worker import WorkThread
 
-FIRST_LINE = "StepUp Queue sbatch wait log format version 1"
+FIRST_LINE = "StepUp Queue sbatch wait log format version 2"
 SCONTROL_FAILED = "The command `scontrol show job` failed!\n"
 DEBUG = string_to_bool(os.getenv("STEPUP_SBATCH_DEBUG", "0"))
 CACHE_TIMEOUT = int(os.getenv("STEPUP_SBATCH_CACHE_TIMEOUT", "30"))
@@ -39,8 +39,27 @@ POLLING_INTERVAL = int(os.getenv("STEPUP_SBATCH_POLLING_INTERVAL", "10"))
 TIME_MARGIN = int(os.getenv("STEPUP_SBATCH_TIME_MARGIN", "5"))
 
 
-def submit_once_and_wait(work_thread: WorkThread):
-    """Submit a job and wait for it to complete. When called a second time, just wait."""
+def submit_once_and_wait(
+    work_thread: WorkThread, job_ext: str, sbatch_rc: str | None = None
+) -> int:
+    """Submit a job and wait for it to complete. When called a second time, just wait.
+
+    Parameters
+    ----------
+    work_thread
+        The work thread to use for launching the subprocesses.
+    job_ext
+        The file extension of the job script to be submitted.
+    sbatch_rc
+        A resource configuration needed before calling sbatch.
+        This is executed in the same shell, right before calling sbatch.
+
+    Returns
+    -------
+    returncode
+        The return code of the job.
+        0 if successful, 1 if the job failed.
+    """
     # Read previously logged steps
     path_log = Path("slurmjob.log")
     if path_log.is_file():
@@ -54,7 +73,7 @@ def submit_once_and_wait(work_thread: WorkThread):
     if status is None:
         # A new job must be submitted.
         submit_time = time.time()
-        sbatch_stdout = submit_job(work_thread)
+        sbatch_stdout = submit_job(work_thread, job_ext, sbatch_rc)
         log_step(path_log, f"Submitted {sbatch_stdout}")
         rndsleep()
     else:
@@ -78,6 +97,13 @@ def submit_once_and_wait(work_thread: WorkThread):
             work_thread, submit_time, jobid, cluster, previous_lines, path_log, status
         )
 
+    # Get the return code from the job
+    with open("slurmjob.ret") as fh:
+        returncode = fh.read().strip()
+    if returncode == "":
+        raise ValueError("The job did not return a return code, e.g. because it was cancelled.")
+    return int(returncode)
+
 
 def _read_log(path_log: str) -> list[str]:
     """Read lines from a previously created log file."""
@@ -87,6 +113,10 @@ def _read_log(path_log: str) -> list[str]:
             check_log_version(next(f).strip())
         except StopIteration as exc:
             raise ValueError("Existing log file is empty.") from exc
+        try:
+            check_log_inp_digest(next(f).strip())
+        except StopIteration as exc:
+            raise ValueError("Existing log file is empty.") from exc
         for line in f:
             line = line.strip()
             lines.append(line)
@@ -95,8 +125,12 @@ def _read_log(path_log: str) -> list[str]:
 
 def _init_log(path_log: str):
     """Initialize a new log file."""
-    with open(path_log, "w") as f:
-        f.write(FIRST_LINE + "\n")
+    inp_digest = os.getenv("STEPUP_STEP_INP_DIGEST")
+    if inp_digest is None:
+        raise ValueError("The environment variable STEPUP_STEP_INP_DIGEST is not set.")
+    with open(path_log, "w") as fh:
+        print(FIRST_LINE, file=fh)
+        print(inp_digest, file=fh)
 
 
 def _read_or_poll_status(
@@ -143,10 +177,10 @@ def _read_or_poll_status(
         # Call scontrol and parse its response.
         rndsleep()
         status_time, status = get_status(work_thread, jobid, cluster)
-        if status is not None and status != last_status:
+        if status != last_status:
             log_step(path_log, status)
     done = (status_time > submit_time + TIME_MARGIN) and (
-        status not in ["PENDING", "CONFIGURING", "RUNNING"]
+        status not in ["PENDING", "CONFIGURING", "RUNNING", "invalid"]
     )
     return status, done
 
@@ -159,6 +193,22 @@ def check_log_version(line: str):
         )
 
 
+class InpDigestError(ValueError):
+    """The input digest in the log file does not match the one in the environment."""
+
+
+def check_log_inp_digest(line: str):
+    """Validate the log input digest, abort if there is a mismatch."""
+    inp_digest = os.getenv("STEPUP_STEP_INP_DIGEST")
+    if inp_digest is None:
+        raise ValueError("The environment variable STEPUP_STEP_INP_DIGEST is not set.")
+    if line != inp_digest:
+        raise InpDigestError(
+            "The second line of the log contains the wrong input digest.\n"
+            f"Expected: {inp_digest}\nFound:    {line}"
+        )
+
+
 def read_step(lines: list[str]) -> str | None:
     """Read a step from the log file."""
     if len(lines) == 0:
@@ -176,10 +226,35 @@ def rndsleep():
     time.sleep(sleep_seconds)
 
 
-def submit_job(work_thread: WorkThread) -> str:
+JOB_SCRIPT_WRAPPER = """\
+#!/usr/bin/env bash
+{sbatch_header}
+
+touch slurmjob.ret
+chmod +x '{job_script}'
+./'{job_script}'
+RETURN_CODE=$?
+echo $RETURN_CODE > slurmjob.ret
+exot $RETURN_CODE
+"""
+
+
+def submit_job(work_thread: WorkThread, job_ext: str, sbatch_rc: str | None = None) -> str:
     """Submit a job with sbatch."""
+    # Copy the #SBATCH lines from the job script.
+    path_job = f"slurmjob{job_ext}"
+    with open(path_job) as f:
+        sbatch_header = "\n".join(line for line in f if line.startswith("#SBATCH"))
+
+    command = "sbatch --parsable -o slurmjob.out -e slurmjob.err"
+    if sbatch_rc is not None:
+        command = f"{sbatch_rc} < /dev/null && {command}"
     returncode, stdout, stderr = work_thread.runsh(
-        "sbatch --parsable -o slurmjob.out -e slurmjob.err slurmjob.sh"
+        command,
+        stdin=JOB_SCRIPT_WRAPPER.format(
+            sbatch_header=sbatch_header,
+            job_script=path_job,
+        ),
     )
     if returncode != 0:
         if not (stderr is None or stderr == ""):
@@ -206,7 +281,7 @@ def parse_sbatch(stdout: str) -> tuple[int, str | None]:
     raise ValueError(f"Cannot parse sbatch output: {stdout}")
 
 
-def get_status(work_thread: WorkThread, jobid: int, cluster: str | None) -> str | None:
+def get_status(work_thread: WorkThread, jobid: int, cluster: str | None) -> str:
     """Load cached scontrol output or run scontrol if outdated.
 
     Parameters
@@ -221,10 +296,9 @@ def get_status(work_thread: WorkThread, jobid: int, cluster: str | None) -> str
     Returns
     -------
     status
-        A status reported by scontrol.
-        The "Invalid" status is returned when scontrol fails to find the jobid.
-        None is returned when scontrol fails in a way that is safe to ignore.
-        (Try again later.)
+        A status reported by scontrol,
+        or `invalid` if scontrol failed (retry scontrol later),
+        or `unlisted` if the job is not found (probably ended long ago).
     """
     # Load cached output or run again
     command = "scontrol show job"
@@ -234,11 +308,17 @@ def get_status(work_thread: WorkThread, jobid: int, cluster: str | None) -> str
     else:
         command += f" --cluster={cluster}"
         path_out /= f"sbatch_wait.{cluster}.out"
-    status_time, scontrol_out = cached_run(work_thread, command, path_out, CACHE_TIMEOUT)
+    status_time, scontrol_out, returncode = cached_run(
+        work_thread, command, path_out, CACHE_TIMEOUT
+    )
+    if returncode != 0:
+        return status_time, "invalid"
     return status_time, parse_scontrol_out(scontrol_out, jobid)
 
 
-def cached_run(work_thread: WorkThread, command: str, path_out: Path, cache_timeout) -> str:
+def cached_run(
+    work_thread: WorkThread, command: str, path_out: Path, cache_timeout
+) -> tuple[float, str, int]:
     """Execute a command if its previous output is outdated.
 
     Parameters
@@ -254,8 +334,12 @@ def cached_run(work_thread: WorkThread, command: str, path_out: Path, cache_time
 
     Returns
     -------
+    cache_time
+        The time when the command was last executed.
     stdout
         The output of the file, either new or cached.
+    returncode
+        The return code of the (cached) command.
 
     Notes
     -----
@@ -270,7 +354,7 @@ def cached_run(work_thread: WorkThread, command: str, path_out: Path, cache_time
         fcntl.lockf(fh, fcntl.LOCK_EX)
         fh.seek(0)
         header = fh.read(CACHE_HEADER_LENGTH)
-        cache_time, _ = parse_cache_header(header)
+        cache_time, returncode = parse_cache_header(header)
         if cache_time is None or time.time() > cache_time + cache_timeout:
             returncode, stdout, _ = work_thread.runsh(command)
             # Go the the beginning of the file before truncating.
@@ -283,8 +367,8 @@ def cached_run(work_thread: WorkThread, command: str, path_out: Path, cache_time
             fh.write(stdout)
             fh.flush()
             os.fsync(fh.fileno())
-            return cache_time, stdout
-        return cache_time, fh.read()
+            return cache_time, stdout, returncode
+        return cache_time, fh.read(), returncode
 
 
 def make_cache_header(cache_time: float, returncode: int):
@@ -311,7 +395,7 @@ def parse_cache_header(header: str) -> tuple[float, int]:
 CACHE_HEADER_LENGTH = len(make_cache_header(time.time(), 0))
 
 
-def parse_scontrol_out(scontrol_out: str, jobid: int) -> str | None:
+def parse_scontrol_out(scontrol_out: str, jobid: int) -> str:
     """Get the job state for a specific from from the output of ``scontrol show job``.
 
     Parameters
@@ -324,10 +408,12 @@ def parse_scontrol_out(scontrol_out: str, jobid: int) -> str | None:
     Returns
     -------
     jobstate
-        The status of the job, or None of the job cannot be found.
+        The status of the job. This can be:
+
+        - Any of the SLURM job states.
+        - `unlisted` if the job cannot be found,
+          which practically means it has ended long ago.
     """
-    if scontrol_out == SCONTROL_FAILED:
-        return "Invalid"
     match = re.search(
         f"JobId={jobid}.*?JobState=(?P<state>[A-Z]+)",
         scontrol_out,
@@ -335,4 +421,4 @@ def parse_scontrol_out(scontrol_out: str, jobid: int) -> str | None:
     )
     if match is not None:
         return match.group("state")
-    return None
+    return "unlisted"

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/stepup_queue.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stepup-queue
-Version: 1.0.1
+Version: 1.0.3
 Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
 Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
 License-Expression: GPL-3.0-or-later
@@ -47,8 +47,9 @@ Dynamic: license-file
 ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
 ![GPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-queue)
 
-StepUp Queue is an experimental extension of
-[StepUp Core](https://reproducible-reporting.github.io/stepup-core)
-to integrate queued jobs into a workflow.
+StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
+extension to integrate queued jobs into a workflow.
 Currently, it only supports integration with [SLURM](https://slurm.schedmd.com/),
 but it is designed to be extensible to other job schedulers.
+
+For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/stepup_queue.egg-info/SOURCES.txt

@@ -16,11 +16,20 @@ docs/development.md
 docs/index.md
 docs/installation.md
 docs/license.md
+docs/stepup.queue.api.md
 docs/usage.md
-docs/examples/slurm/.gitignore
-docs/examples/slurm/dynamic-template.sh
-docs/examples/slurm/plan.py
-docs/examples/slurm/static/slurmjob.sh
+docs/examples/slurm-basic/.gitignore
+docs/examples/slurm-basic/README.md
+docs/examples/slurm-basic/dynamic-template.sh
+docs/examples/slurm-basic/plan.py
+docs/examples/slurm-basic/fail/slurmjob.sh
+docs/examples/slurm-basic/pass/slurmjob.py
+docs/examples/slurm-perpetual/.gitignore
+docs/examples/slurm-perpetual/README.md
+docs/examples/slurm-perpetual/plan.py
+docs/examples/slurm-perpetual/workflow.sh
+docs/examples/slurm-perpetual/step1/slurmjob.sh
+docs/examples/slurm-perpetual/step2/slurmjob.sh
 overrides/main.html
 stepup/queue/__init__.py
 stepup/queue/actions.py

{stepup_queue-1.0.1 → stepup_queue-1.0.3}/tests/test_sbatch.py

@@ -56,15 +56,17 @@ def test_parse_sbatch():
 def test_cached_run(path_tmp: Path):
     path_out = path_tmp / "date.txt"
     work_thread = WorkThread("<test>")
-    cache_time1, out1 = cached_run(work_thread, "date", path_out, 1)
-    cache_time2, out2 = cached_run(work_thread, "date", path_out, 10)
+    cache_time1, out1, ret1 = cached_run(work_thread, "date", path_out, 1)
+    cache_time2, out2, ret2 = cached_run(work_thread, "date", path_out, 10)
     assert cache_time1 == pytest.approx(cache_time2, 1e-4)
     assert out1 != ""
     assert out1 == out2
+    assert ret1 == ret2
     time.sleep(2)
-    cache_time3, out3 = cached_run(work_thread, "date", path_out, 1)
+    cache_time3, out3, ret3 = cached_run(work_thread, "date", path_out, 1)
     assert abs(cache_time1 - cache_time3) > 0.5
     assert out1 != out3
+    assert ret1 == ret3
 
 
 SCONTROL_OUT = """\

stepup_queue-1.0.1/docs/examples/slurm/plan.py

@@ -1,15 +0,0 @@
-#!/usr/bin/env python3
-
-from stepup.core.api import mkdir, render_jinja, static
-from stepup.queue.api import sbatch
-
-# First an example of a static job script, i.e. already present on disk.
-static("static/", "static/slurmjob.sh")
-sbatch("static")
-
-# Now an example of a job script that is generated from a template.
-static("dynamic-template.sh")
-for i in range(1, 4):
-    mkdir(f"dynamic{i}/")
-    render_jinja("dynamic-template.sh", {"field": i}, f"dynamic{i}/slurmjob.sh")
-    sbatch(f"dynamic{i}/")

stepup_queue-1.0.1/docs/examples/slurm/static/slurmjob.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-#SBATCH -J static
-#SBATCH -N 1
-
-echo "Hello from static job"
-sleep 5
-echo "Goodbye from static job"

stepup_queue-1.0.1/stepup/queue/actions.py

@@ -1,31 +0,0 @@
-# StepUp Queue integrates queued jobs into a StepUp workflow.
-# © 2025 Toon Verstraelen
-#
-# This file is part of StepUp Queue.
-#
-# StepUp Queue is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# StepUp Queue is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses/>
-#
-# --
-"""StepUp Queue package."""
-
-from stepup.core.worker import WorkThread
-
-from .sbatch import submit_once_and_wait
-
-
-def sbatch(argstr: str, work_thread: WorkThread) -> int:
-    if argstr != "":
-        raise ValueError("sbatch does not accept any arguments")
-    submit_once_and_wait(work_thread)
-    return 0