stepup-queue 2.0.1__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/CLAUDE.md +15 -0
  2. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/MANIFEST.in +1 -1
  3. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/PKG-INFO +6 -8
  4. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/README.md +1 -1
  5. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/REUSE.toml +0 -5
  6. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/pyproject.toml +11 -8
  7. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/snipwise.md +2 -2
  8. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/CLAUDE.md +3 -0
  9. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/api.py +21 -1
  10. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/cancel_jobs.py +2 -1
  11. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/log.py +26 -4
  12. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/remove_jobs.py +3 -3
  13. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/sbatch.py +74 -11
  14. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/utils.py +2 -1
  15. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup_queue.egg-info/PKG-INFO +6 -8
  16. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup_queue.egg-info/requires.txt +3 -6
  17. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup_queue.egg-info/scm_file_list.json +5 -4
  18. stepup_queue-2.1.0/stepup_queue.egg-info/scm_version.json +8 -0
  19. stepup_queue-2.0.1/stepup_queue.egg-info/scm_version.json +0 -8
  20. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/.zenodo.json +0 -0
  21. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/CITATION.cff +0 -0
  22. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/LICENSE +0 -0
  23. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/LICENSES/CC-BY-SA-4.0.txt +0 -0
  24. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/LICENSES/LGPL-3.0-or-later.txt +0 -0
  25. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
  26. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/setup.cfg +0 -0
  27. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup/queue/__init__.py +0 -0
  28. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup_queue.egg-info/SOURCES.txt +0 -0
  29. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup_queue.egg-info/dependency_links.txt +0 -0
  30. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup_queue.egg-info/entry_points.txt +0 -0
  31. {stepup_queue-2.0.1 → stepup_queue-2.1.0}/stepup_queue.egg-info/top_level.txt +0 -0
@@ -45,6 +45,21 @@ The package is organized around two layers:
45
45
  Use "which"/"because"/"that", or split the sentence.
46
46
  - **Do not add `# noqa`**
47
47
  unless the violation is a genuine false positive that cannot be resolved by restructuring.
48
+ - **A path is a string with `/` as its only separator, also on Windows.**
49
+ Import `Path` from `stepup.core.path`, not from `path`, and avoid `os.path`.
50
+ `test_slash_paths` of `ConventionTests` enforces this.
51
+ StepUp Queue only supports POSIX systems,
52
+ but it follows the path and text file conventions of StepUp Core all the same.
53
+
54
+ ### Text Files
55
+
56
+ Open every text file with `encoding="utf-8"`, and write text files with `newline="\n"` as well,
57
+ so that files are identical on all platforms,
58
+ including Windows with Python 3.14 and older, where the default is the ANSI code page.
59
+ Ruff enforces the encoding with `PLW1514`,
60
+ but only for `open()` and `pathlib`, not for the methods of `Path`.
61
+ In the package, `read_text()` and `write_text()` of `Path` need `encoding="utf-8"` too.
62
+ The same holds for `subprocess.run()` with `text=True`, which ruff does not check either.
48
63
 
49
64
  ### Docstrings
50
65
 
@@ -9,7 +9,7 @@ prune overrides
9
9
  exclude .editorconfig
10
10
  exclude .gitignore
11
11
  exclude .pre-commit-config.yaml
12
- exclude mkdocs.yaml
12
+ exclude zensical.toml
13
13
 
14
14
  # Exclude tests for now. (Could be useful later for conda package.)
15
15
  prune tests
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup-queue
3
- Version: 2.0.1
3
+ Version: 2.1.0
4
4
  Summary: Integrate SLURM jobs into a StepUp workflow
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
@@ -21,13 +21,13 @@ Classifier: Programming Language :: Python :: 3.11
21
21
  Classifier: Programming Language :: Python :: 3.12
22
22
  Classifier: Programming Language :: Python :: 3.13
23
23
  Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Programming Language :: Python :: 3.15
24
25
  Classifier: Topic :: Software Development :: Build Tools
25
26
  Requires-Python: >=3.11
26
27
  Description-Content-Type: text/markdown
27
28
  License-File: LICENSE
28
- Requires-Dist: path>=16.14.0
29
29
  Requires-Dist: rich>=13.0.0
30
- Requires-Dist: stepup<5.0.0a1,>=4.0.3
30
+ Requires-Dist: stepup<5.0.0a1,>=4.1.0
31
31
  Provides-Extra: tests
32
32
  Requires-Dist: pytest>=8.2.0; extra == "tests"
33
33
  Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
@@ -35,12 +35,10 @@ Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
35
35
  Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
36
36
  Provides-Extra: dev
37
37
  Requires-Dist: stepup-queue[tests]; extra == "dev"
38
- Requires-Dist: mkdocs; extra == "dev"
39
- Requires-Dist: mkdocstrings[python]; extra == "dev"
40
- Requires-Dist: mkdocs-macros-plugin; extra == "dev"
41
- Requires-Dist: mkdocs-material; extra == "dev"
38
+ Requires-Dist: mkdocstrings-python; extra == "dev"
42
39
  Requires-Dist: reuse; extra == "dev"
43
40
  Requires-Dist: snipwise<2.0a0; extra == "dev"
41
+ Requires-Dist: zensical; extra == "dev"
44
42
  Dynamic: license-file
45
43
 
46
44
  <!--
@@ -52,7 +50,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
52
50
 
53
51
  [![release](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml)
54
52
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
55
- [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
53
+ [![docs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml)
56
54
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
57
55
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
58
56
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
@@ -7,7 +7,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
7
7
 
8
8
  [![release](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml)
9
9
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
10
- [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
10
+ [![docs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml)
11
11
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
12
12
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
13
13
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
@@ -14,11 +14,6 @@ path = [
14
14
  SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
15
15
  SPDX-License-Identifier = "LGPL-3.0-or-later"
16
16
 
17
- [[annotations]]
18
- path = "docs/clean_stdout.sed"
19
- SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
20
- SPDX-License-Identifier = "LGPL-3.0-or-later"
21
-
22
17
  [[annotations]]
23
18
  path = [
24
19
  "docs/logo.svg",
@@ -52,12 +52,12 @@ classifiers = [
52
52
  "Programming Language :: Python :: 3.12",
53
53
  "Programming Language :: Python :: 3.13",
54
54
  "Programming Language :: Python :: 3.14",
55
+ "Programming Language :: Python :: 3.15",
55
56
  "Topic :: Software Development :: Build Tools",
56
57
  ]
57
58
  dependencies = [
58
- "path>=16.14.0",
59
59
  "rich>=13.0.0",
60
- "stepup>=4.0.3,<5.0.0a1",
60
+ "stepup>=4.1.0,<5.0.0a1",
61
61
  ]
62
62
  dynamic = ["version"]
63
63
 
@@ -74,12 +74,10 @@ tests = [
74
74
  ]
75
75
  dev = [
76
76
  "stepup-queue[tests]",
77
- "mkdocs",
78
- "mkdocstrings[python]",
79
- "mkdocs-macros-plugin",
80
- "mkdocs-material",
77
+ "mkdocstrings-python",
81
78
  "reuse",
82
79
  "snipwise<2.0a0",
80
+ "zensical",
83
81
  ]
84
82
 
85
83
  [project.urls]
@@ -103,9 +101,14 @@ line-length = 100
103
101
  target-version = "py311"
104
102
 
105
103
  [tool.ruff.lint]
104
+ # Preview mode is only enabled for the preview rules selected by their full code below,
105
+ # but it also turns on the preview behavior of stable rules.
106
+ preview = true
107
+ explicit-preview-rules = true
106
108
  select = [
107
109
  "A", "B", "BLE", "C4", "E", "EXE", "F", "I", "ICN", "ISC", "N", "NPY", "PERF", "PIE", "PL",
108
- "PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W"
110
+ "PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W",
111
+ "PLW1514", # https://docs.astral.sh/ruff/rules/unspecified-encoding/ (preview in ruff 0.17.2)
109
112
  ]
110
113
  ignore = [
111
114
  "PLR0904", # https://docs.astral.sh/ruff/rules/too-many-public-methods/
@@ -142,7 +145,7 @@ mode.strict-config = true
142
145
  # The documentation pages carry YAML front matter,
143
146
  # which pymarkdown parses as a setext heading unless this extension is on.
144
147
  extensions.front-matter.enabled = true
145
- # The documentation indents nested lists by four spaces, matching Python-Markdown and MkDocs.
148
+ # The documentation indents nested lists by four spaces, matching Python-Markdown and Zensical.
146
149
  plugins.md007.indent = 4
147
150
  plugins.md013.line_length = 100
148
151
  plugins.md013.heading_line_length = 100
@@ -22,9 +22,9 @@ render = "{{ content | unwrap }}"
22
22
 
23
23
  # The meta description that every documentation page carries.
24
24
  [[targets]]
25
- patterns = ["mkdocs.yaml"]
25
+ patterns = ["zensical.toml"]
26
26
  scanner = "regex"
27
- regex = '(?m)^site_description: (?P<content>.*)$'
27
+ regex = '(?m)^site_description = "(?P<content>[^"]*)"$'
28
28
  snippets = ["tagline"]
29
29
  render = "{{ content | unwrap }}"
30
30
 
@@ -31,6 +31,9 @@ attaches to the job it already submitted, instead of submitting a second one:
31
31
  3. With a log whose digest matches, it resumes waiting for the existing job.
32
32
  4. With a digest mismatch, the `onchange` policy decides
33
33
  between `raise`, `resubmit` and `ignore`.
34
+ With `raise` or `resubmit`, an optional `complete` command is consulted first.
35
+ When it succeeds, the new digest is written into the log and the existing job is kept,
36
+ which brings the step back to case 3.
34
37
 
35
38
  This is why the digest is written into the log rather than derived at read time:
36
39
  the log has to be interpretable by a process that never saw the original submission.
@@ -21,6 +21,7 @@ def sbatch(
21
21
  out: Collection[StrPath] | StrPath = (),
22
22
  vol: Collection[StrPath] | StrPath = (),
23
23
  onchange: str | None = None,
24
+ complete: str | None = None,
24
25
  optional: bool = False,
25
26
  resources: dict[str, int] | str | None = None,
26
27
  duration: float | None = None,
@@ -63,8 +64,23 @@ def sbatch(
63
64
  and pass it here as `source rc.sh`.
64
65
  In this case, you usually also want to include `rc.sh` in the `inp` list.
65
66
  onchange
66
- Policy when a the inputs of a previously submitted job have changed.
67
+ Policy when the inputs of a previously submitted job have changed.
67
68
  Must be one of `"raise"`, `"resubmit"` or `"ignore"`.
69
+ When not given, the environment variable `STEPUP_QUEUE_ONCHANGE` is used,
70
+ with `"raise"` as default.
71
+ complete
72
+ A shell command that tells whether the outputs of the job are complete,
73
+ e.g. `"test -f result.txt"`.
74
+ The command is executed in the working directory
75
+ when the inputs of a previously submitted job have changed.
76
+ A zero exit code keeps the existing job, as if its inputs had not changed.
77
+ A non-zero exit code applies the `onchange` policy.
78
+ Hence, with `onchange="resubmit"`, a job whose inputs have changed is only resubmitted
79
+ when outputs are missing.
80
+ This is useful for job scripts that skip work whose outputs already exist.
81
+ With `onchange="raise"`, an error is only raised when outputs are missing.
82
+ The command is not used with `onchange="ignore"`,
83
+ because that policy keeps the existing job anyway.
68
84
  """
69
85
  if ext == "":
70
86
  ext = ".sh"
@@ -81,6 +97,10 @@ def sbatch(
81
97
  if onchange not in ["raise", "resubmit", "ignore"]:
82
98
  raise ValueError(f"Invalid onchange policy {onchange}.")
83
99
  cmd += f" --onchange={onchange}"
100
+ if complete is not None:
101
+ if complete.strip() == "":
102
+ raise ValueError("The complete command must not be empty.")
103
+ cmd += f" --complete={shlex.quote(complete)}"
84
104
  return run(
85
105
  cmd,
86
106
  inp=[f"slurmjob{ext}", *coerce_paths(inp)],
@@ -7,9 +7,10 @@ import subprocess
7
7
  import sys
8
8
  from collections.abc import Sequence
9
9
 
10
- from path import Path
11
10
  from rich.console import Console
12
11
 
12
+ from stepup.core.path import Path
13
+
13
14
  from .log import read_jobid_cluster_status
14
15
  from .utils import DONE_STATES, search_jobs
15
16
 
@@ -2,9 +2,10 @@
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
3
  """The job log file format and utilities to read and write it."""
4
4
 
5
+ import os
5
6
  from datetime import datetime
6
7
 
7
- from path import Path
8
+ from stepup.core.path import Path
8
9
 
9
10
  from .utils import parse_sbatch
10
11
 
@@ -18,6 +19,7 @@ __all__ = (
18
19
  "read_jobid_cluster_status",
19
20
  "read_log",
20
21
  "read_status",
22
+ "update_log_inp_digest",
21
23
  )
22
24
 
23
25
  FIRST_LINE = "StepUp Queue sbatch wait log format version 2"
@@ -42,15 +44,35 @@ class InterruptedSubmissionError(ValueError):
42
44
 
43
45
  def init_log(path_log: str, inp_digest: str):
44
46
  """Initialize a new log file."""
45
- with open(path_log, "w") as fh:
47
+ with open(path_log, "w", encoding="utf-8", newline="\n") as fh:
46
48
  print(FIRST_LINE, file=fh)
47
49
  print(inp_digest, file=fh)
48
50
 
49
51
 
52
+ def update_log_inp_digest(path_log: str, inp_digest: str):
53
+ """Replace the input digest in an existing log file, keeping all status lines.
54
+
55
+ The new log is written to a temporary file first,
56
+ so that an interruption cannot leave a truncated log behind.
57
+ """
58
+ path_log = Path(path_log)
59
+ with open(path_log, encoding="utf-8") as fh:
60
+ lines = fh.readlines()
61
+ if len(lines) < 2:
62
+ raise ValueError(f"Existing log file has no input digest: {path_log}")
63
+ check_log_version(lines[0].strip())
64
+ lines[1] = f"{inp_digest}\n"
65
+ path_tmp = path_log.with_name(f".{path_log.name}.tmp")
66
+ with open(path_tmp, "w", encoding="utf-8", newline="\n") as fh:
67
+ fh.writelines(lines)
68
+ # `os.replace` is used because `Path.replace` is `str.replace` in older versions of `path`.
69
+ os.replace(path_tmp, path_log)
70
+
71
+
50
72
  def log_status(path_log: Path, status: str):
51
73
  """Write a status to the log."""
52
74
  dt = datetime.now().isoformat()
53
- with open(path_log, "a") as f:
75
+ with open(path_log, "a", encoding="utf-8", newline="\n") as f:
54
76
  line = f"{dt} {status}"
55
77
  f.write(f"{line}\n")
56
78
 
@@ -91,7 +113,7 @@ def read_jobid_cluster_status(path_log: str) -> tuple[int, str | None, str | Non
91
113
  def read_log(path_log: str, expected_inp_digest: str | None = None) -> list[str]:
92
114
  """Read lines from a previously created log file."""
93
115
  lines = []
94
- with open(path_log) as f:
116
+ with open(path_log, encoding="utf-8") as f:
95
117
  try:
96
118
  check_log_version(next(f).strip())
97
119
  except StopIteration as exc:
@@ -3,14 +3,14 @@
3
3
  """Remove the directories of failed jobs."""
4
4
 
5
5
  import argparse
6
- import os
7
6
  import shutil
8
7
  import sys
9
8
  from collections.abc import Sequence
10
9
 
11
- from path import Path
12
10
  from rich.console import Console
13
11
 
12
+ from stepup.core.path import Path
13
+
14
14
  from .log import read_log, read_status
15
15
  from .utils import search_jobs
16
16
 
@@ -60,7 +60,7 @@ def remove_jobs(argv: Sequence[str] | None = None):
60
60
  if path_full == path_cwd:
61
61
  console.print("[red]# WARNING: Refusing to remove the current directory.[/]")
62
62
  continue
63
- if path_cwd.startswith(os.path.join(path_full, "")):
63
+ if path_cwd.startswith(path_full / ""):
64
64
  console.print(
65
65
  f"[red]# WARNING: Refusing to remove {path_job}, "
66
66
  "a parent of the current directory.[/]"
@@ -3,7 +3,6 @@
3
3
  """An sbatch wrapper to submit only on the first call, and to wait until a job has finished."""
4
4
 
5
5
  import argparse
6
- import fcntl
7
6
  import os
8
7
  import random
9
8
  import re
@@ -14,10 +13,8 @@ import time
14
13
  from collections.abc import Sequence
15
14
  from datetime import datetime
16
15
 
17
- from path import Path
18
-
19
16
  from stepup.core.extapi import record_subprocess, run_subprocess
20
- from stepup.core.path import get_root
17
+ from stepup.core.path import Path, get_root
21
18
 
22
19
  from .log import (
23
20
  InpDigestError,
@@ -28,9 +25,18 @@ from .log import (
28
25
  read_jobid_cluster_status,
29
26
  read_log,
30
27
  read_status,
28
+ update_log_inp_digest,
31
29
  )
32
30
  from .utils import DONE_STATES, KNOWN_JOB_STATES, parse_sbatch
33
31
 
32
+ try:
33
+ import fcntl
34
+ except ImportError as exc:
35
+ raise ImportError(
36
+ "sq-sbatch-and-wait needs a POSIX system with the fcntl module. "
37
+ "Workflows that submit SLURM jobs run on Linux, on macOS or in WSL."
38
+ ) from exc
39
+
34
40
  __all__ = ("sbatch",)
35
41
 
36
42
 
@@ -47,6 +53,59 @@ UNLISTED_TIMEOUT = int(os.getenv("STEPUP_SBATCH_UNLISTED_TIMEOUT", "600"))
47
53
  CANCEL_TIMEOUT = int(os.getenv("STEPUP_SBATCH_CANCEL_TIMEOUT", "600"))
48
54
 
49
55
 
56
+ def get_inp_digest() -> str:
57
+ """Return the input digest of the step, which StepUp passes through the environment."""
58
+ inp_digest = os.getenv("STEPUP_STEP_INP_DIGEST")
59
+ if inp_digest is None:
60
+ raise ValueError("The environment variable STEPUP_STEP_INP_DIGEST is not set.")
61
+ return inp_digest
62
+
63
+
64
+ def keep_complete_job(complete: str) -> bool:
65
+ """Keep a previously submitted job despite changed inputs, if its outputs are complete.
66
+
67
+ Parameters
68
+ ----------
69
+ complete
70
+ A shell command that is executed in the job directory.
71
+ A zero exit code means that the outputs of the step are complete.
72
+ It is only executed when the log records a submitted job
73
+ whose input digest differs from the current one.
74
+
75
+ Returns
76
+ -------
77
+ kept
78
+ True if the input digest in the log was replaced by the current one,
79
+ so that the existing job is treated as if it had been submitted with the current inputs.
80
+ False if the inputs did not change, if no job was submitted,
81
+ or if the command reports incomplete outputs.
82
+ """
83
+ inp_digest = get_inp_digest()
84
+ path_log = Path("slurmjob.log")
85
+ if not path_log.is_file():
86
+ return False
87
+ try:
88
+ read_log(path_log, inp_digest)
89
+ except InpDigestError:
90
+ pass
91
+ else:
92
+ return False
93
+ try:
94
+ jobid, _, _ = read_jobid_cluster_status(path_log)
95
+ except (NoSubmissionError, InterruptedSubmissionError):
96
+ # Without a recorded job, there is nothing to keep.
97
+ # The onchange policy decides what happens next.
98
+ return False
99
+ cp = run_subprocess(complete, shell=True, check=False)
100
+ if cp.returncode != 0:
101
+ print(f"Inputs changed and outputs are incomplete according to: {complete}")
102
+ return False
103
+ update_log_inp_digest(path_log, inp_digest)
104
+ print(f"Inputs changed, but outputs are complete according to: {complete}")
105
+ print(f"Keeping job {jobid} without resubmission.")
106
+ return True
107
+
108
+
50
109
  def submit_once_and_wait(
51
110
  job_ext: str,
52
111
  sbatch_rc: str | None = None,
@@ -65,9 +124,7 @@ def submit_once_and_wait(
65
124
  If False, the input digest is not checked.
66
125
  This is useful when the job script is modified but the changes are harmless.
67
126
  """
68
- inp_digest = os.getenv("STEPUP_STEP_INP_DIGEST")
69
- if inp_digest is None:
70
- raise ValueError("The environment variable STEPUP_STEP_INP_DIGEST is not set.")
127
+ inp_digest = get_inp_digest()
71
128
  start_time = time.time()
72
129
 
73
130
  # Read previously logged job states
@@ -134,7 +191,7 @@ def submit_once_and_wait(
134
191
 
135
192
  if status == "COMPLETED":
136
193
  # Get the return code from the job
137
- with open("slurmjob.ret") as fh:
194
+ with open("slurmjob.ret", encoding="utf-8") as fh:
138
195
  returncode = fh.read().strip()
139
196
  try:
140
197
  returncode = int(returncode)
@@ -250,7 +307,7 @@ def submit_job(job_ext: str, sbatch_rc: str | None = None) -> str:
250
307
  raise ValueError("The job script must be executable.")
251
308
 
252
309
  # Copy the #SBATCH lines from the job script and perform some checks.
253
- with open(path_job) as f:
310
+ with open(path_job, encoding="utf-8") as f:
254
311
  sbatch_header = []
255
312
  first_line = next(f, "")
256
313
  if not first_line.startswith("#!"):
@@ -375,14 +432,16 @@ def cached_run(
375
432
  path_out.absolute().parent.makedirs_p()
376
433
  path_out.touch()
377
434
 
378
- with open(path_out, mode="r+") as fh:
435
+ with open(path_out, mode="r+", encoding="utf-8", newline="\n") as fh:
379
436
  fcntl.lockf(fh, fcntl.LOCK_EX)
380
437
  fh.seek(0)
381
438
  header = fh.read(CACHE_HEADER_LENGTH)
382
439
  cache_time, returncode = parse_cache_header(header)
383
440
  if cache_time is None or time.time() > cache_time + cache_timeout:
384
441
  stdout = fh.read() if cache_time is not None else ""
385
- cp = subprocess.run(shlex.split(command), capture_output=True, text=True, check=False)
442
+ cp = subprocess.run(
443
+ shlex.split(command), capture_output=True, text=True, encoding="utf-8", check=False
444
+ )
386
445
  if first:
387
446
  # Only the first call is recorded to avoid duplicate entries in StepUp's metadata.
388
447
  # Note that the recording of subprocesses is intended to be informative,
@@ -535,8 +594,12 @@ def sbatch(argv: Sequence[str] | None = None):
535
594
  parser.add_argument(
536
595
  "--onchange", default=default_onchange, choices=["raise", "resubmit", "ignore"]
537
596
  )
597
+ parser.add_argument("--complete", default=None)
538
598
  args = parser.parse_args(argv)
539
599
 
600
+ if args.complete is not None and args.onchange != "ignore":
601
+ keep_complete_job(args.complete)
602
+
540
603
  if args.onchange == "resubmit":
541
604
  try:
542
605
  submit_once_and_wait(args.ext, args.rc)
@@ -4,9 +4,10 @@
4
4
 
5
5
  from itertools import chain
6
6
 
7
- from path import Path
8
7
  from rich.console import Console
9
8
 
9
+ from stepup.core.path import Path
10
+
10
11
  __all__ = (
11
12
  "DONE_STATES",
12
13
  "KNOWN_JOB_STATES",
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup-queue
3
- Version: 2.0.1
3
+ Version: 2.1.0
4
4
  Summary: Integrate SLURM jobs into a StepUp workflow
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
@@ -21,13 +21,13 @@ Classifier: Programming Language :: Python :: 3.11
21
21
  Classifier: Programming Language :: Python :: 3.12
22
22
  Classifier: Programming Language :: Python :: 3.13
23
23
  Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Programming Language :: Python :: 3.15
24
25
  Classifier: Topic :: Software Development :: Build Tools
25
26
  Requires-Python: >=3.11
26
27
  Description-Content-Type: text/markdown
27
28
  License-File: LICENSE
28
- Requires-Dist: path>=16.14.0
29
29
  Requires-Dist: rich>=13.0.0
30
- Requires-Dist: stepup<5.0.0a1,>=4.0.3
30
+ Requires-Dist: stepup<5.0.0a1,>=4.1.0
31
31
  Provides-Extra: tests
32
32
  Requires-Dist: pytest>=8.2.0; extra == "tests"
33
33
  Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
@@ -35,12 +35,10 @@ Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
35
35
  Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
36
36
  Provides-Extra: dev
37
37
  Requires-Dist: stepup-queue[tests]; extra == "dev"
38
- Requires-Dist: mkdocs; extra == "dev"
39
- Requires-Dist: mkdocstrings[python]; extra == "dev"
40
- Requires-Dist: mkdocs-macros-plugin; extra == "dev"
41
- Requires-Dist: mkdocs-material; extra == "dev"
38
+ Requires-Dist: mkdocstrings-python; extra == "dev"
42
39
  Requires-Dist: reuse; extra == "dev"
43
40
  Requires-Dist: snipwise<2.0a0; extra == "dev"
41
+ Requires-Dist: zensical; extra == "dev"
44
42
  Dynamic: license-file
45
43
 
46
44
  <!--
@@ -52,7 +50,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
52
50
 
53
51
  [![release](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml)
54
52
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
55
- [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
53
+ [![docs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml)
56
54
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
57
55
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
58
56
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
@@ -1,15 +1,12 @@
1
- path>=16.14.0
2
1
  rich>=13.0.0
3
- stepup<5.0.0a1,>=4.0.3
2
+ stepup<5.0.0a1,>=4.1.0
4
3
 
5
4
  [dev]
6
5
  stepup-queue[tests]
7
- mkdocs
8
- mkdocstrings[python]
9
- mkdocs-macros-plugin
10
- mkdocs-material
6
+ mkdocstrings-python
11
7
  reuse
12
8
  snipwise<2.0a0
9
+ zensical
13
10
 
14
11
  [tests]
15
12
  pytest>=8.2.0
@@ -2,7 +2,7 @@
2
2
  "files": [
3
3
  ".editorconfig",
4
4
  ".github/extract-notes.sh",
5
- ".github/workflows/mkdocs.yaml",
5
+ ".github/workflows/docs.yaml",
6
6
  ".github/workflows/pytest.yaml",
7
7
  ".github/workflows/release.yaml",
8
8
  ".gitignore",
@@ -24,6 +24,7 @@
24
24
  "docs/examples/slurm-basic/README.md",
25
25
  "docs/examples/slurm-basic/dynamic-template.sh",
26
26
  "docs/examples/slurm-basic/fail/slurmjob.sh",
27
+ "docs/examples/slurm-basic/parts-template.sh",
27
28
  "docs/examples/slurm-basic/pass/slurmjob.py",
28
29
  "docs/examples/slurm-basic/plan.py",
29
30
  "docs/examples/slurm-perpetual/.gitignore",
@@ -40,8 +41,6 @@
40
41
  "docs/social-card.svg",
41
42
  "docs/stepup.queue.api.md",
42
43
  "docs/usage.md",
43
- "mkdocs.yaml",
44
- "overrides/404.html",
45
44
  "overrides/main.html",
46
45
  "pyproject.toml",
47
46
  "snipwise.md",
@@ -55,11 +54,13 @@
55
54
  "stepup/queue/utils.py",
56
55
  "tests/CLAUDE.md",
57
56
  "tests/conftest.py",
57
+ "tests/test_api.py",
58
58
  "tests/test_cancel_jobs.py",
59
59
  "tests/test_conventions.py",
60
60
  "tests/test_log.py",
61
61
  "tests/test_remove_jobs.py",
62
62
  "tests/test_sbatch.py",
63
- "tests/test_utils.py"
63
+ "tests/test_utils.py",
64
+ "zensical.toml"
64
65
  ]
65
66
  }
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "2.1.0",
3
+ "distance": 0,
4
+ "node": "g1d677ef83a06bbc0c290eb513a78261249b24de6",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-11-20"
8
+ }
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "2.0.1",
3
- "distance": 0,
4
- "node": "g77c01c8c740b15aa91320d9fb2d909ba6877f6c3",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-09-19"
8
- }
File without changes
File without changes
File without changes
File without changes