stepup-queue 2.0.0rc2__tar.gz → 2.0.0rc4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. stepup_queue-2.0.0rc4/.zenodo.json +52 -0
  2. stepup_queue-2.0.0rc4/CLAUDE.md +160 -0
  3. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/MANIFEST.in +0 -1
  4. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/PKG-INFO +5 -4
  5. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/README.md +2 -2
  6. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/REUSE.toml +5 -0
  7. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/pyproject.toml +17 -1
  8. stepup_queue-2.0.0rc4/stepup/queue/CLAUDE.md +44 -0
  9. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup/queue/__init__.py +2 -0
  10. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup/queue/api.py +2 -0
  11. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup/queue/canceljobs.py +2 -0
  12. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup/queue/removejobs.py +3 -0
  13. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup/queue/sbatch.py +3 -0
  14. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup_queue.egg-info/PKG-INFO +5 -4
  15. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup_queue.egg-info/SOURCES.txt +2 -0
  16. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup_queue.egg-info/requires.txt +2 -1
  17. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup_queue.egg-info/scm_file_list.json +39 -36
  18. stepup_queue-2.0.0rc4/stepup_queue.egg-info/scm_version.json +8 -0
  19. stepup_queue-2.0.0rc2/CLAUDE.md +0 -130
  20. stepup_queue-2.0.0rc2/stepup_queue.egg-info/scm_version.json +0 -8
  21. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/LICENSE +0 -0
  22. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/LICENSES/CC-BY-SA-4.0.txt +0 -0
  23. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/LICENSES/LGPL-3.0-or-later.txt +0 -0
  24. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
  25. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/setup.cfg +0 -0
  26. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup/queue/log.py +0 -0
  27. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup/queue/utils.py +0 -0
  28. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup_queue.egg-info/dependency_links.txt +0 -0
  29. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup_queue.egg-info/entry_points.txt +0 -0
  30. {stepup_queue-2.0.0rc2 → stepup_queue-2.0.0rc4}/stepup_queue.egg-info/top_level.txt +0 -0
@@ -0,0 +1,52 @@
1
+ {
2
+ "upload_type": "software",
3
+ "title": "StepUp Queue",
4
+ "description": "StepUp Queue is an experimental StepUp extension to integrate queued jobs into a workflow. It currently supports integration with SLURM and is designed to be extensible to other job schedulers.",
5
+ "creators": [
6
+ {
7
+ "name": "Verstraelen, Toon",
8
+ "orcid": "0000-0001-9288-5608",
9
+ "affiliation": "Center for Molecular Modeling (CMM), Ghent University"
10
+ }
11
+ ],
12
+ "license": "lgpl-3.0-or-later",
13
+ "access_right": "open",
14
+ "communities": [
15
+ {
16
+ "identifier": "ugent"
17
+ }
18
+ ],
19
+ "language": "eng",
20
+ "keywords": [
21
+ "SLURM",
22
+ "job scheduler",
23
+ "job submission",
24
+ "queued jobs",
25
+ "batch computing",
26
+ "high-performance computing",
27
+ "supercomputer",
28
+ "workflow automation",
29
+ "workflow management",
30
+ "build tool",
31
+ "build automation",
32
+ "reproducibility",
33
+ "reproducible research",
34
+ "research software",
35
+ "extension",
36
+ "StepUp Core",
37
+ "POSIX",
38
+ "Python"
39
+ ],
40
+ "related_identifiers": [
41
+ {
42
+ "identifier": "https://reproducible-reporting.github.io/stepup-queue",
43
+ "relation": "isDocumentedBy",
44
+ "resource_type": "publication-softwaredocumentation"
45
+ },
46
+ {
47
+ "identifier": "10.5281/zenodo.22018869",
48
+ "relation": "requires",
49
+ "resource_type": "software"
50
+ }
51
+ ]
52
+ }
@@ -0,0 +1,160 @@
1
+ <!--
2
+ SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
+ SPDX-License-Identifier: LGPL-3.0-or-later
4
+ -->
5
+
6
+ # CLAUDE.md
7
+
8
+ This file provides guidance to Claude Code (claude.ai/code)
9
+ when working with code in this repository.
10
+
11
+ Guidance that applies to only part of the repo lives next to the code it governs:
12
+
13
+ - `stepup/queue/CLAUDE.md`: job file conventions, the submit-and-wait contract, sacct caching.
14
+ - `tests/CLAUDE.md`: test layout and conventions.
15
+
16
+ ## Project Overview
17
+
18
+ StepUp Queue is a [StepUp Core](https://github.com/reproducible-reporting/stepup-core)
19
+ extension that integrates the SLURM job scheduler into a StepUp workflow.
20
+ It lets workflows submit SLURM jobs, wait for them,
21
+ and resume from existing jobs after a restart,
22
+ which makes long-running HPC workflows resumable across interrupted sessions.
23
+
24
+ The related `stepup-core` repo is at `../stepup-core` and on GitHub.
25
+
26
+ ## Architecture
27
+
28
+ The package is organized around two layers:
29
+
30
+ - **`stepup/queue/api.py`** is the Python API that users call in their `plan.py` build scripts.
31
+ Its `sbatch()` function calls `stepup.core.api.run()`
32
+ to register a `sq-sbatch-and-wait` step with StepUp Core.
33
+
34
+ - **Individual command modules** each implement a `main()` function that serves as a CLI tool.
35
+ These are registered via `[project.scripts]` and the `stepup.tools` entry points
36
+ in `pyproject.toml`, and are invoked by StepUp steps as external commands.
37
+ When StepUp executes such a step, the command runs in the working directory of the job.
38
+
39
+ ## Coding Conventions
40
+
41
+ - **En and em dashes** never appear in prose (comments, docstrings, Markdown),
42
+ in neither glyph (–, —) nor ASCII (--) form.
43
+ Use "which"/"because"/"that", or split the sentence.
44
+ - **Do not add `# noqa`**
45
+ unless the violation is a genuine false positive that cannot be resolved by restructuring.
46
+
47
+ ### Docstrings
48
+
49
+ Use **NumPy-style** sections (`Parameters`, `Returns`, `Raises`, ...)
50
+ Some conventions specific to this codebase:
51
+
52
+ - Docstrings are written in Markdown, not reStructuredText! Some important gotcha's:
53
+ - Do not use italics for parameter names, return values, or exception names.
54
+ Use single backticks instead.
55
+ - Use single backticks for all inline code, not double backticks.
56
+ - Use triple backticks for code blocks,
57
+ and specify the language for syntax highlighting (e.g., ```python).
58
+ - Lines are wrapped using semantic breaks, per [Semantic Line Breaks](#semantic-line-breaks) below.
59
+ - Use the imperative mood for function descriptions
60
+ (e.g., "Compute the hash of a file."),
61
+ except for `@property` getters where the description should be a noun phrase
62
+ (e.g., "The parent directory path.").
63
+ - Do not repeat type annotations in the docstring, they are already in the function signature.
64
+ - In `Parameters` sections, use the **parameter name** as the heading for each parameter.
65
+ Grouping closely related parameters under a combined heading
66
+ (e.g., `stdout, stderr`) is allowed when parameters are better described together.
67
+
68
+ - In `Returns` sections, use a **semantic name** for the return value, not the type,
69
+ as these are already in the function signature.
70
+
71
+ ```python
72
+ # correct
73
+ Returns
74
+ -------
75
+ parent
76
+ The parent directory path.
77
+
78
+ # wrong, the type is already in the signature
79
+ Returns
80
+ -------
81
+ Path
82
+ The parent directory path.
83
+ ```
84
+
85
+ ### Markdown
86
+
87
+ Section headings (`##`, `###`, ...) use **Title Case**
88
+ (capitalize nouns, verbs, adjectives, and adverbs; lowercase articles,
89
+ coordinating conjunctions, and prepositions regardless of length, e.g. "from", "with").
90
+ Inline code spans (e.g. `` `sbatch()` ``) keep their own casing and are never title-cased.
91
+
92
+ ### Semantic Line Breaks
93
+
94
+ All English prose in this repo (comments, docstrings, Markdown documentation, commit messages, ...)
95
+ uses **semantic line breaks**.
96
+ See <https://sembr.org/>.
97
+ Prose diffs then stay small, because editing one sentence never reflows its neighbours.
98
+
99
+ - **Every sentence starts on a new line.**
100
+ - **Break inside a sentence only where a break is needed, and then at a clause boundary.**
101
+ A sentence that fits within the 100-character line length stays on a single line.
102
+ A longer one is broken before a conjunction or a relative pronoun
103
+ ("and", "but", "because", "which", "if", ...),
104
+ or after a leading subordinate clause.
105
+ - **Not every comma is a break.**
106
+ Enumerated items, appositions and short parentheticals stay on the line they started on.
107
+
108
+ The 100-character line length is a hard cap, not a target to fill.
109
+ An extra break inside a long sentence is fine when it clarifies the structure,
110
+ but a sentence that already fits on one line is left alone.
111
+
112
+ ### Prose That Ages Well
113
+
114
+ Stale prose is worse than no prose.
115
+ When writing comments, docstrings, or other prose, avoid:
116
+
117
+ - **Describing callers.** Don't note how other code uses a function or class.
118
+ That's the caller's concern, and the remark silently rots when the caller changes.
119
+ - **Describing history.** Don't explain what the code used to do or how it changed.
120
+ The current code should speak for itself; history belongs in commit messages.
121
+ - **Implementation details in docstrings.** Document the contract (how to use something),
122
+ not how it works internally.
123
+ - **Line-number references.** They break as soon as the file changes.
124
+ Point to a function, class, or file name instead.
125
+ - **Restating the code.** A comment should say something the code doesn't already say
126
+ (the reason, the invariant, the non-obvious constraint) not paraphrase the next line.
127
+ A purely redundant comment isn't wrong, so nothing forces it to be updated,
128
+ and it drifts out of sync silently.
129
+ - **Repetitive and duplicate comments.**
130
+ If a remark is repeated in multiple places, it will rot in one place when updated in another.
131
+ Factor out the common remark into a single function or class docstring,
132
+ or a Markdown file in `docs/`.
133
+ - **Timeless phrasing for point-in-time claims.**
134
+ An empirical observation about an external tool or environment
135
+ (e.g. "sacct always reports a terminal state") can stop being true after a version upgrade,
136
+ with nothing to flag the comment as outdated.
137
+ Say what was observed and, when it matters, on what
138
+ (e.g. "as of SLURM 23.11, observed on a single-cluster setup").
139
+
140
+ ### `__all__`
141
+
142
+ Wildcard imports are banned (ruff `F403`), so `__all__` does not describe a star-import
143
+ surface here. It is the module's **import contract**: the names that code outside the module
144
+ is meant to import.
145
+
146
+ - Every module in `stepup/queue/` declares `__all__`, placed directly after the imports
147
+ and before any module-level constant.
148
+ It is a tuple of string literals, sorted (enforced by ruff `RUF022`),
149
+ and it is declared exactly once per module.
150
+ - List a name when it is imported by another `stepup` module, a downstream extension package,
151
+ a user's `plan.py`, or a `pyproject.toml` entry point (e.g. `build_subcommand`).
152
+ - Do not list module-internal names, even when they lack a leading underscore:
153
+ `logger`, helpers used only within the module.
154
+ A public-looking name is not a claim that the name is exported.
155
+ - Tests may import names that are not in `__all__`; white-box testing does not make a name
156
+ part of the contract.
157
+ - Do not re-export: a name in `__all__` must be defined in that same module.
158
+ Import a name from the module that defines it, not from a module that happens to import it.
159
+ - `__all__ = ()` is a real claim, nothing outside the module may import from it,
160
+ and is correct only for leaf modules.
@@ -8,7 +8,6 @@ prune docs
8
8
  prune overrides
9
9
  exclude .editorconfig
10
10
  exclude .gitignore
11
- exclude .markdownlint-cli2.jsonc
12
11
  exclude .pre-commit-config.yaml
13
12
  exclude mkdocs.yaml
14
13
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup-queue
3
- Version: 2.0.0rc2
3
+ Version: 2.0.0rc4
4
4
  Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
@@ -26,8 +26,9 @@ Description-Content-Type: text/markdown
26
26
  License-File: LICENSE
27
27
  Requires-Dist: path>=16.14.0
28
28
  Requires-Dist: rich>=13.0.0
29
- Requires-Dist: stepup<5.0.0a1,>=4.0.0rc8
29
+ Requires-Dist: stepup<5.0.0a1,>=4.0.0rc12
30
30
  Provides-Extra: dev
31
+ Requires-Dist: cffconvert; extra == "dev"
31
32
  Requires-Dist: mike; extra == "dev"
32
33
  Requires-Dist: mkdocs; extra == "dev"
33
34
  Requires-Dist: mkdocstrings[python]; extra == "dev"
@@ -44,7 +45,6 @@ Dynamic: license-file
44
45
  SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
45
46
  SPDX-License-Identifier: LGPL-3.0-or-later
46
47
  -->
47
- <!-- markdownlint-disable line-length -->
48
48
 
49
49
  # StepUp Queue
50
50
 
@@ -52,8 +52,9 @@ SPDX-License-Identifier: LGPL-3.0-or-later
52
52
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
53
53
  [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
54
54
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
55
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
55
56
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
56
- ![LGPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-queue)
57
+ ![LGPL-3 License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)
57
58
 
58
59
  StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
59
60
  extension to integrate queued jobs into a workflow.
@@ -2,7 +2,6 @@
2
2
  SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
3
  SPDX-License-Identifier: LGPL-3.0-or-later
4
4
  -->
5
- <!-- markdownlint-disable line-length -->
6
5
 
7
6
  # StepUp Queue
8
7
 
@@ -10,8 +9,9 @@ SPDX-License-Identifier: LGPL-3.0-or-later
10
9
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
11
10
  [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
12
11
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
12
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
13
13
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
14
- ![LGPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-queue)
14
+ ![LGPL-3 License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)
15
15
 
16
16
  StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
17
17
  extension to integrate queued jobs into a workflow.
@@ -2,6 +2,11 @@
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
3
  version = 1
4
4
 
5
+ [[annotations]]
6
+ path = ".zenodo.json"
7
+ SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
8
+ SPDX-License-Identifier = "LGPL-3.0-or-later"
9
+
5
10
  [[annotations]]
6
11
  path = [
7
12
  "docs/examples/*/**/*",
@@ -32,12 +32,13 @@ dependencies = [
32
32
  # Ensure changes to these dependencies are reflected in .github/requirements-old.txt
33
33
  "path>=16.14.0",
34
34
  "rich>=13.0.0",
35
- "stepup>=4.0.0rc8,<5.0.0a1",
35
+ "stepup>=4.0.0rc12,<5.0.0a1",
36
36
  ]
37
37
  dynamic = ["version"]
38
38
 
39
39
  [project.optional-dependencies]
40
40
  dev = [
41
+ "cffconvert",
41
42
  "mike",
42
43
  "mkdocs",
43
44
  "mkdocstrings[python]",
@@ -104,3 +105,18 @@ packages = ["stepup.queue"]
104
105
  [tool.setuptools_scm]
105
106
  version_scheme = "post-release"
106
107
  local_scheme = "no-local-version"
108
+
109
+ [tool.pymarkdown]
110
+ # Report unknown or mistyped settings instead of silently ignoring them.
111
+ mode.strict-config = true
112
+ # The documentation indents nested lists by four spaces, matching Python-Markdown and MkDocs.
113
+ plugins.md007.indent = 4
114
+ plugins.md013.line_length = 100
115
+ plugins.md013.heading_line_length = 100
116
+ plugins.md013.code_block_line_length = 100
117
+ # The changelog repeats headings such as "Fixed" under every release,
118
+ # which is fine as long as they do not repeat under the same release.
119
+ plugins.md024.siblings_only = true
120
+ # Rule md041 (first-line-heading) counts the SPDX header in an HTML comment
121
+ # as content preceding the first heading.
122
+ plugins.md041.enabled = false
@@ -0,0 +1,44 @@
1
+ <!--
2
+ SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
+ SPDX-License-Identifier: LGPL-3.0-or-later
4
+ -->
5
+
6
+ # Job Files and the Submit-and-Wait Contract
7
+
8
+ ## Job Lifecycle and Files
9
+
10
+ Every SLURM job lives in its own working directory, with these file conventions:
11
+
12
+ - `slurmjob{ext}`: the user-written job script.
13
+ It must be executable and must have a shebang.
14
+ - `slurmjob.log`: the StepUp Queue log, tracking submission and SLURM state history.
15
+ - `slurmjob.out` and `slurmjob.err`: SLURM stdout and stderr, declared as `out`.
16
+ - `slurmjob.ret`: the exit code written by the wrapper script, declared as `out`.
17
+
18
+ `slurmjob.log` is declared as a `vol` (volatile) file in StepUp, not as `out`,
19
+ because its contents are timestamps and scheduler state,
20
+ which are not reproducible output.
21
+
22
+ ## Idempotent Submit-and-Wait
23
+
24
+ `submit_once_and_wait()` in `sbatch.py` is the core function.
25
+ The contract it implements is that a step which is interrupted and re-run
26
+ attaches to the job it already submitted, instead of submitting a second one:
27
+
28
+ 1. It reads `slurmjob.log` and compares the stored input digest
29
+ against `STEPUP_STEP_INP_DIGEST`.
30
+ 2. With no log, it submits a new job.
31
+ 3. With a log whose digest matches, it resumes waiting for the existing job.
32
+ 4. With a digest mismatch, the `onchange` policy decides
33
+ between `raise`, `resubmit` and `ignore`.
34
+
35
+ This is why the digest is written into the log rather than derived at read time:
36
+ the log has to be interpretable by a process that never saw the original submission.
37
+
38
+ ## sacct Caching
39
+
40
+ All concurrent `sq-sbatch-and-wait` processes share one cached `sacct` result per cluster,
41
+ guarded by an `fcntl.LOCK_EX` lock, so that only one process queries SLURM at a time.
42
+ Without this, a workflow with many parallel jobs hammers the scheduler
43
+ with one `sacct` call per job per polling interval,
44
+ which is the kind of load that gets a user throttled or banned on a shared cluster.
@@ -1,3 +1,5 @@
1
1
  # SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
3
  """StepUp Queue package."""
4
+
5
+ __all__ = ()
@@ -23,6 +23,7 @@ def sbatch(
23
23
  onchange: str | None = None,
24
24
  optional: bool = False,
25
25
  resources: dict[str, int] | str | None = None,
26
+ duration: float | None = None,
26
27
  ):
27
28
  """Submit a SLURM job script.
28
29
 
@@ -89,4 +90,5 @@ def sbatch(
89
90
  workdir=workdir,
90
91
  optional=optional,
91
92
  resources=resources,
93
+ duration=duration,
92
94
  )
@@ -15,6 +15,8 @@ from stepup.core.config import ConfigLoader
15
15
  from .log import read_jobid_cluster_status
16
16
  from .utils import DONE_STATES, search_jobs
17
17
 
18
+ __all__ = ()
19
+
18
20
 
19
21
  def canceljobs_tool(args: argparse.Namespace):
20
22
  """Iterate over all slurmjob.log files, read the SLURM job IDs, and cancel them."""
@@ -14,6 +14,9 @@ from stepup.core.config import ConfigLoader
14
14
  from .log import read_log, read_status
15
15
  from .utils import search_jobs
16
16
 
17
+ __all__ = ()
18
+
19
+
17
20
  FAILED_STATES = {
18
21
  "BOOT_FAIL",
19
22
  "CANCELLED",
@@ -27,6 +27,9 @@ from .log import (
27
27
  )
28
28
  from .utils import DONE_STATES, KNOWN_JOB_STATES, parse_sbatch
29
29
 
30
+ __all__ = ()
31
+
32
+
30
33
  SBATCH_RETRY_NUM = int(os.getenv("STEPUP_SBATCH_RETRY_NUM", "5"))
31
34
  SBATCH_RETRY_DELAY_MIN = int(os.getenv("STEPUP_SBATCH_RETRY_DELAY_MIN", "60"))
32
35
  SBATCH_RETRY_DELAY_MAX = int(os.getenv("STEPUP_SBATCH_RETRY_DELAY_MAX", "120"))
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup-queue
3
- Version: 2.0.0rc2
3
+ Version: 2.0.0rc4
4
4
  Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
@@ -26,8 +26,9 @@ Description-Content-Type: text/markdown
26
26
  License-File: LICENSE
27
27
  Requires-Dist: path>=16.14.0
28
28
  Requires-Dist: rich>=13.0.0
29
- Requires-Dist: stepup<5.0.0a1,>=4.0.0rc8
29
+ Requires-Dist: stepup<5.0.0a1,>=4.0.0rc12
30
30
  Provides-Extra: dev
31
+ Requires-Dist: cffconvert; extra == "dev"
31
32
  Requires-Dist: mike; extra == "dev"
32
33
  Requires-Dist: mkdocs; extra == "dev"
33
34
  Requires-Dist: mkdocstrings[python]; extra == "dev"
@@ -44,7 +45,6 @@ Dynamic: license-file
44
45
  SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
45
46
  SPDX-License-Identifier: LGPL-3.0-or-later
46
47
  -->
47
- <!-- markdownlint-disable line-length -->
48
48
 
49
49
  # StepUp Queue
50
50
 
@@ -52,8 +52,9 @@ SPDX-License-Identifier: LGPL-3.0-or-later
52
52
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
53
53
  [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
54
54
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
55
+ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
55
56
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
56
- ![LGPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-queue)
57
+ ![LGPL-3 License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)
57
58
 
58
59
  StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
59
60
  extension to integrate queued jobs into a workflow.
@@ -1,3 +1,4 @@
1
+ .zenodo.json
1
2
  CLAUDE.md
2
3
  LICENSE
3
4
  MANIFEST.in
@@ -7,6 +8,7 @@ pyproject.toml
7
8
  LICENSES/CC-BY-SA-4.0.txt
8
9
  LICENSES/LGPL-3.0-or-later.txt
9
10
  LICENSES/LicenseRef-StepUp-Logo-v1.txt
11
+ stepup/queue/CLAUDE.md
10
12
  stepup/queue/__init__.py
11
13
  stepup/queue/api.py
12
14
  stepup/queue/canceljobs.py
@@ -1,8 +1,9 @@
1
1
  path>=16.14.0
2
2
  rich>=13.0.0
3
- stepup<5.0.0a1,>=4.0.0rc8
3
+ stepup<5.0.0a1,>=4.0.0rc12
4
4
 
5
5
  [dev]
6
+ cffconvert
6
7
  mike
7
8
  mkdocs
8
9
  mkdocstrings[python]
@@ -1,55 +1,58 @@
1
1
  {
2
2
  "files": [
3
- ".pre-commit-config.yaml",
4
- "REUSE.toml",
5
- "README.md",
6
- "LICENSE",
7
- "pyproject.toml",
8
- "mkdocs.yaml",
9
- "CLAUDE.md",
10
- "MANIFEST.in",
11
3
  ".editorconfig",
4
+ ".github/requirements-old.txt",
5
+ ".github/scripts/extract-notes.sh",
6
+ ".github/workflows/mkdocs.yaml",
7
+ ".github/workflows/pytest.yaml",
8
+ ".github/workflows/release.yaml",
12
9
  ".gitignore",
13
- ".markdownlint-cli2.jsonc",
14
- "LICENSES/LicenseRef-StepUp-Logo-v1.txt",
15
- "LICENSES/LGPL-3.0-or-later.txt",
10
+ ".pre-commit-config.yaml",
11
+ ".zenodo.json",
12
+ "CLAUDE.md",
13
+ "LICENSE",
16
14
  "LICENSES/CC-BY-SA-4.0.txt",
17
- "docs/development.md",
18
- "docs/installation.md",
19
- "docs/license.md",
15
+ "LICENSES/LGPL-3.0-or-later.txt",
16
+ "LICENSES/LicenseRef-StepUp-Logo-v1.txt",
17
+ "MANIFEST.in",
18
+ "README.md",
19
+ "REUSE.toml",
20
20
  "docs/changelog.md",
21
- "docs/usage.md",
22
- "docs/logo.svg",
23
- "docs/stepup.queue.api.md",
24
- "docs/index.md",
21
+ "docs/development.md",
22
+ "docs/examples/slurm-basic/.gitignore",
23
+ "docs/examples/slurm-basic/README.md",
24
+ "docs/examples/slurm-basic/dynamic-template.sh",
25
+ "docs/examples/slurm-basic/fail/slurmjob.sh",
26
+ "docs/examples/slurm-basic/pass/slurmjob.py",
27
+ "docs/examples/slurm-basic/plan.py",
28
+ "docs/examples/slurm-perpetual/.gitignore",
25
29
  "docs/examples/slurm-perpetual/README.md",
26
- "docs/examples/slurm-perpetual/workflow.sh",
27
30
  "docs/examples/slurm-perpetual/plan.py",
28
- "docs/examples/slurm-perpetual/.gitignore",
29
31
  "docs/examples/slurm-perpetual/step1/slurmjob.sh",
30
32
  "docs/examples/slurm-perpetual/step2/slurmjob.sh",
31
- "docs/examples/slurm-basic/README.md",
32
- "docs/examples/slurm-basic/dynamic-template.sh",
33
- "docs/examples/slurm-basic/plan.py",
34
- "docs/examples/slurm-basic/.gitignore",
35
- "docs/examples/slurm-basic/pass/slurmjob.py",
36
- "docs/examples/slurm-basic/fail/slurmjob.sh",
33
+ "docs/examples/slurm-perpetual/workflow.sh",
34
+ "docs/index.md",
35
+ "docs/installation.md",
36
+ "docs/license.md",
37
+ "docs/logo.svg",
38
+ "docs/stepup.queue.api.md",
39
+ "docs/usage.md",
40
+ "mkdocs.yaml",
37
41
  "overrides/main.html",
42
+ "pyproject.toml",
43
+ "stepup/queue/CLAUDE.md",
38
44
  "stepup/queue/__init__.py",
39
- "stepup/queue/sbatch.py",
40
- "stepup/queue/utils.py",
41
- "stepup/queue/canceljobs.py",
42
45
  "stepup/queue/api.py",
46
+ "stepup/queue/canceljobs.py",
43
47
  "stepup/queue/log.py",
44
48
  "stepup/queue/removejobs.py",
45
- "tests/test_utils.py",
49
+ "stepup/queue/sbatch.py",
50
+ "stepup/queue/utils.py",
51
+ "tests/CLAUDE.md",
52
+ "tests/conftest.py",
53
+ "tests/test_conventions.py",
46
54
  "tests/test_log.py",
47
55
  "tests/test_sbatch.py",
48
- "tests/conftest.py",
49
- ".github/requirements-old.txt",
50
- ".github/scripts/extract-notes.sh",
51
- ".github/workflows/pytest.yaml",
52
- ".github/workflows/release.yaml",
53
- ".github/workflows/mkdocs.yaml"
56
+ "tests/test_utils.py"
54
57
  ]
55
58
  }
@@ -0,0 +1,8 @@
1
+ {
2
+ "tag": "2.0.0rc4",
3
+ "distance": 0,
4
+ "node": "g6c139922f2238a206f1ee71763a0b3083777cf15",
5
+ "dirty": false,
6
+ "branch": "HEAD",
7
+ "node_date": "2026-08-25"
8
+ }
@@ -1,130 +0,0 @@
1
- <!--
2
- SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
- SPDX-License-Identifier: LGPL-3.0-or-later
4
- -->
5
- # CLAUDE.md
6
-
7
- This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
8
-
9
- ## Project Overview
10
-
11
- StepUp Queue is a StepUp Core extension that integrates SLURM job scheduler workflows. It allows
12
- StepUp workflows to submit SLURM jobs, wait for them, and resume from existing jobs after restarts
13
- — making long-running HPC workflows resumable across interrupted sessions.
14
-
15
- The related `stepup-core` repo is at `../stepup-core` and on GitHub.
16
-
17
- ## Development Environment
18
-
19
- Uses [uv](https://docs.astral.sh/uv/) for environment management:
20
-
21
- ```bash
22
- uv sync --extra dev
23
- pre-commit install
24
- direnv allow # activates .venv and sets env vars from .envrc
25
- ```
26
-
27
- The `.envrc` sets `STEPUP_DEBUG=1`, `STEPUP_BUILD_DURATION=0`, and `STEPUP_SYNC_RPC_TIMEOUT=30`.
28
- Without `direnv`, prefix commands with `uv run`.
29
-
30
- ## Common Commands
31
-
32
- ```bash
33
- # Run all tests (parallel by default via pytest-xdist, quite fast)
34
- pytest -vv
35
-
36
- # Run all linters
37
- pre-commit run --all
38
-
39
- # Docs live preview
40
- mkdocs serve
41
- ```
42
-
43
- ## Architecture
44
-
45
- ### Package layout
46
-
47
- ```text
48
- stepup/queue/
49
- api.py — Public Python API: sbatch() for use in plan.py files
50
- sbatch.py — sq-sbatch-and-wait CLI: submits, waits, polls, caches sacct output
51
- log.py — slurmjob.log format (version 2): read/write/validate
52
- utils.py — SLURM state sets, parse_sbatch(), search_jobs()
53
- canceljobs.py — stepup canceljobs subcommand
54
- removejobs.py — stepup removejobs subcommand
55
- ```
56
-
57
- ### How it fits into StepUp
58
-
59
- `stepup.queue.api.sbatch()` is called from a user's `plan.py`. It calls
60
- `stepup.core.api.run()` to register the `sq-sbatch-and-wait` step with StepUp Core.
61
- When StepUp executes that step, `sq-sbatch-and-wait` (entry point for `stepup/queue/sbatch.py`)
62
- runs in the working directory of the job.
63
-
64
- ### Job lifecycle and files
65
-
66
- Every SLURM job lives in its own working directory. The conventions are:
67
-
68
- - `slurmjob{ext}` — the user-written job script (must be executable, must have shebang)
69
- - `slurmjob.log` — StepUp Queue's log (volatile; tracks submission + SLURM state history)
70
- - `slurmjob.out` / `slurmjob.err` — SLURM stdout/stderr (declared as `out`)
71
- - `slurmjob.ret` — exit code written by wrapper script (declared as `out`)
72
-
73
- `slurmjob.log` is declared as a `vol` (volatile) file in StepUp, not `out`, so it is not
74
- treated as reproducible output. It contains: a version header, an input digest (SHA-256 of
75
- all step inputs), and timestamped status lines (`Submitted <jobid>[;cluster]`, then SLURM states).
76
-
77
- ### Idempotent submit-and-wait
78
-
79
- `submit_once_and_wait()` in `sbatch.py` is the core function:
80
-
81
- 1. Reads `slurmjob.log` and checks the stored input digest against `STEPUP_STEP_INP_DIGEST`.
82
- 2. If no log exists → submits a new job via `sbatch --parsable`.
83
- 3. If log exists with a matching digest → resumes waiting for the existing job.
84
- 4. If digest mismatch → behaviour depends on `onchange` policy (`raise`/`resubmit`/`ignore`).
85
- 5. Polls status via `sacct`, using a **shared on-disk cache** at
86
- `.stepup/queue/sbatch_wait_sacct[.cluster].out` with `fcntl.LOCK_EX` to avoid
87
- hammering SLURM when many jobs run in parallel.
88
-
89
- ### sacct caching
90
-
91
- `cached_run()` in `sbatch.py` manages the shared `sacct` cache. All concurrent `sq-sbatch-and-wait`
92
- processes share a single cached file per cluster; only one process calls `sacct` at a time (via
93
- `fcntl` lock). The cache file has a fixed-length header (`v1 datetime=... returncode=...`).
94
-
95
- ### Entry points
96
-
97
- - `sq-sbatch-and-wait` — CLI that wraps `sbatch()` → `submit_once_and_wait()`
98
- - `stepup canceljobs` — registered as `stepup.tools` entry point; cancels running SLURM jobs
99
- by reading `slurmjob.log` files recursively
100
- - `stepup removejobs` — registered as `stepup.tools` entry point; removes directories of failed jobs
101
-
102
- ### Key environment variables
103
-
104
- | Variable | Default | Purpose |
105
- | --- | --- | --- |
106
- | `STEPUP_SBATCH_CACHE_TIMEOUT` | 30 | Seconds between sacct calls |
107
- | `STEPUP_SBATCH_POLLING_MIN/MAX` | 10/20 | Random polling interval (seconds) |
108
- | `STEPUP_SBATCH_RETRY_NUM` | 5 | sbatch retry attempts on transient failure |
109
- | `STEPUP_SBATCH_RETRY_DELAY_MIN/MAX` | 60/120 | Retry delay range (seconds) |
110
- | `STEPUP_SACCT_START_TIME` | now-7days | `-S` argument passed to sacct |
111
- | `STEPUP_SBATCH_UNLISTED_TIMEOUT` | 600 | Seconds before unlisted job is declared failed |
112
- | `STEPUP_QUEUE_ONCHANGE` | raise | Default `onchange` policy |
113
-
114
- ### Linting
115
-
116
- Ruff with `line-length = 100`, targeting Python 3.11+. The `ruff.lint` section in
117
- `pyproject.toml` selects many rule sets; several `PLR` (complexity) rules are deliberately
118
- disabled. Imports are sorted with `stepup` as a known-first-party package.
119
-
120
- ### Testing
121
-
122
- `pytest` is configured with `-n auto --dist worksteal -W error` — all warnings are errors,
123
- tests run in parallel. The `conftest.py` provides only a `path_tmp` fixture wrapping `tmpdir`.
124
- Tests are pure unit tests; no SLURM cluster is required.
125
-
126
- ## Release Process
127
-
128
- 1. Update `docs/changelog.md` with the new version.
129
- 2. Commit and tag: `git tag vX.Y.Z`.
130
- 3. Push with tags: `git push origin main --tags` (triggers PyPI GitHub Action).
@@ -1,8 +0,0 @@
1
- {
2
- "tag": "2.0.0rc2",
3
- "distance": 0,
4
- "node": "gbe62ab4e89c254c12836a5d23e786d13a37d2c93",
5
- "dirty": false,
6
- "branch": "HEAD",
7
- "node_date": "2026-07-04"
8
- }
File without changes