stepup-queue 2.0.0rc4__tar.gz → 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/.zenodo.json +15 -11
- stepup_queue-2.1.0/CITATION.cff +68 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/CLAUDE.md +21 -4
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/MANIFEST.in +1 -1
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/PKG-INFO +19 -17
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/README.md +5 -3
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/REUSE.toml +5 -6
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/pyproject.toml +57 -21
- stepup_queue-2.1.0/snipwise.md +118 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/CLAUDE.md +14 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/__init__.py +1 -1
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/api.py +21 -1
- stepup_queue-2.0.0rc4/stepup/queue/canceljobs.py → stepup_queue-2.1.0/stepup/queue/cancel_jobs.py +20 -18
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/log.py +66 -12
- stepup_queue-2.0.0rc4/stepup/queue/removejobs.py → stepup_queue-2.1.0/stepup/queue/remove_jobs.py +33 -14
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/sbatch.py +223 -54
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/utils.py +5 -3
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/PKG-INFO +19 -17
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/SOURCES.txt +4 -2
- stepup_queue-2.1.0/stepup_queue.egg-info/entry_points.txt +4 -0
- stepup_queue-2.1.0/stepup_queue.egg-info/requires.txt +15 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/scm_file_list.json +15 -7
- stepup_queue-2.1.0/stepup_queue.egg-info/scm_version.json +8 -0
- stepup_queue-2.0.0rc4/stepup_queue.egg-info/entry_points.txt +0 -6
- stepup_queue-2.0.0rc4/stepup_queue.egg-info/requires.txt +0 -16
- stepup_queue-2.0.0rc4/stepup_queue.egg-info/scm_version.json +0 -8
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSE +0 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSES/CC-BY-SA-4.0.txt +0 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSES/LGPL-3.0-or-later.txt +0 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/setup.cfg +0 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/dependency_links.txt +0 -0
- {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/top_level.txt +0 -0
|
@@ -18,24 +18,28 @@
|
|
|
18
18
|
],
|
|
19
19
|
"language": "eng",
|
|
20
20
|
"keywords": [
|
|
21
|
-
"
|
|
21
|
+
"asyncio",
|
|
22
|
+
"batch computing",
|
|
23
|
+
"build automation",
|
|
24
|
+
"build tool",
|
|
25
|
+
"extension",
|
|
26
|
+
"high-performance computing",
|
|
27
|
+
"HPC",
|
|
22
28
|
"job scheduler",
|
|
23
29
|
"job submission",
|
|
30
|
+
"parallel",
|
|
31
|
+
"POSIX",
|
|
32
|
+
"Python",
|
|
24
33
|
"queued jobs",
|
|
25
|
-
"batch computing",
|
|
26
|
-
"high-performance computing",
|
|
27
|
-
"supercomputer",
|
|
28
|
-
"workflow automation",
|
|
29
|
-
"workflow management",
|
|
30
|
-
"build tool",
|
|
31
|
-
"build automation",
|
|
32
34
|
"reproducibility",
|
|
33
35
|
"reproducible research",
|
|
34
36
|
"research software",
|
|
35
|
-
"
|
|
37
|
+
"SLURM",
|
|
38
|
+
"StepUp",
|
|
36
39
|
"StepUp Core",
|
|
37
|
-
"
|
|
38
|
-
"
|
|
40
|
+
"supercomputer",
|
|
41
|
+
"workflow automation",
|
|
42
|
+
"workflow management"
|
|
39
43
|
],
|
|
40
44
|
"related_identifiers": [
|
|
41
45
|
{
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
2
|
+
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
3
|
+
cff-version: 1.2.0
|
|
4
|
+
message: If you use this software, please cite it using these metadata.
|
|
5
|
+
title: StepUp Queue
|
|
6
|
+
type: software
|
|
7
|
+
url: https://reproducible-reporting.github.io/stepup-queue
|
|
8
|
+
abstract: >-
|
|
9
|
+
StepUp Queue is an experimental StepUp
|
|
10
|
+
extension to integrate queued jobs into a workflow.
|
|
11
|
+
It currently supports integration with SLURM
|
|
12
|
+
and is designed to be extensible to other job schedulers.
|
|
13
|
+
authors:
|
|
14
|
+
- family-names: Verstraelen
|
|
15
|
+
given-names: Toon
|
|
16
|
+
orcid: https://orcid.org/0000-0001-9288-5608
|
|
17
|
+
affiliation: Center for Molecular Modeling (CMM), Ghent University
|
|
18
|
+
address: Technologiepark-Zwijnaarde 46
|
|
19
|
+
post-code: 9052
|
|
20
|
+
city: Zwijnaarde
|
|
21
|
+
country: BE
|
|
22
|
+
license: LGPL-3.0-or-later
|
|
23
|
+
repository-code: https://github.com/reproducible-reporting/stepup-queue
|
|
24
|
+
keywords:
|
|
25
|
+
# snipwise.md BEGIN keywords
|
|
26
|
+
- asyncio
|
|
27
|
+
- batch computing
|
|
28
|
+
- build automation
|
|
29
|
+
- build tool
|
|
30
|
+
- extension
|
|
31
|
+
- high-performance computing
|
|
32
|
+
- HPC
|
|
33
|
+
- job scheduler
|
|
34
|
+
- job submission
|
|
35
|
+
- parallel
|
|
36
|
+
- POSIX
|
|
37
|
+
- Python
|
|
38
|
+
- queued jobs
|
|
39
|
+
- reproducibility
|
|
40
|
+
- reproducible research
|
|
41
|
+
- research software
|
|
42
|
+
- SLURM
|
|
43
|
+
- StepUp
|
|
44
|
+
- StepUp Core
|
|
45
|
+
- supercomputer
|
|
46
|
+
- workflow automation
|
|
47
|
+
- workflow management
|
|
48
|
+
# snipwise.md END keywords
|
|
49
|
+
identifiers:
|
|
50
|
+
- type: doi
|
|
51
|
+
value: 10.5281/zenodo.22024209
|
|
52
|
+
description: Concept DOI, which always resolves to the latest archived version.
|
|
53
|
+
references:
|
|
54
|
+
- title: StepUp Core
|
|
55
|
+
type: software
|
|
56
|
+
authors:
|
|
57
|
+
- family-names: Verstraelen
|
|
58
|
+
given-names: Toon
|
|
59
|
+
orcid: https://orcid.org/0000-0001-9288-5608
|
|
60
|
+
affiliation: Center for Molecular Modeling (CMM), Ghent University
|
|
61
|
+
url: https://reproducible-reporting.github.io/stepup-core
|
|
62
|
+
repository-code: https://github.com/reproducible-reporting/stepup-core
|
|
63
|
+
license: LGPL-3.0-or-later
|
|
64
|
+
identifiers:
|
|
65
|
+
- type: doi
|
|
66
|
+
value: 10.5281/zenodo.22018869
|
|
67
|
+
description: Concept DOI, which always resolves to the latest archived version.
|
|
68
|
+
notes: StepUp Queue is an extension of StepUp Core and requires it at runtime.
|
|
@@ -12,6 +12,7 @@ Guidance that applies to only part of the repo lives next to the code it governs
|
|
|
12
12
|
|
|
13
13
|
- `stepup/queue/CLAUDE.md`: job file conventions, the submit-and-wait contract, sacct caching.
|
|
14
14
|
- `tests/CLAUDE.md`: test layout and conventions.
|
|
15
|
+
- `docs/CLAUDE.md`: regenerating the social card.
|
|
15
16
|
|
|
16
17
|
## Project Overview
|
|
17
18
|
|
|
@@ -31,10 +32,11 @@ The package is organized around two layers:
|
|
|
31
32
|
Its `sbatch()` function calls `stepup.core.api.run()`
|
|
32
33
|
to register a `sq-sbatch-and-wait` step with StepUp Core.
|
|
33
34
|
|
|
34
|
-
- **Individual command modules** each implement a
|
|
35
|
-
These are registered via `[project.scripts]`
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
- **Individual command modules** each implement a function that serves as a CLI entry point.
|
|
36
|
+
These are registered via `[project.scripts]` in `pyproject.toml`.
|
|
37
|
+
`sq-sbatch-and-wait` is invoked by StepUp steps as an external command,
|
|
38
|
+
and runs in the working directory of the job.
|
|
39
|
+
`sq-cancel-jobs` and `sq-remove-jobs` are run by the user from the shell.
|
|
38
40
|
|
|
39
41
|
## Coding Conventions
|
|
40
42
|
|
|
@@ -43,6 +45,21 @@ The package is organized around two layers:
|
|
|
43
45
|
Use "which"/"because"/"that", or split the sentence.
|
|
44
46
|
- **Do not add `# noqa`**
|
|
45
47
|
unless the violation is a genuine false positive that cannot be resolved by restructuring.
|
|
48
|
+
- **A path is a string with `/` as its only separator, also on Windows.**
|
|
49
|
+
Import `Path` from `stepup.core.path`, not from `path`, and avoid `os.path`.
|
|
50
|
+
`test_slash_paths` of `ConventionTests` enforces this.
|
|
51
|
+
StepUp Queue only supports POSIX systems,
|
|
52
|
+
but it follows the path and text file conventions of StepUp Core all the same.
|
|
53
|
+
|
|
54
|
+
### Text Files
|
|
55
|
+
|
|
56
|
+
Open every text file with `encoding="utf-8"`, and write text files with `newline="\n"` as well,
|
|
57
|
+
so that files are identical on all platforms,
|
|
58
|
+
including Windows with Python 3.14 and older, where the default is the ANSI code page.
|
|
59
|
+
Ruff enforces the encoding with `PLW1514`,
|
|
60
|
+
but only for `open()` and `pathlib`, not for the methods of `Path`.
|
|
61
|
+
In the package, `read_text()` and `write_text()` of `Path` need `encoding="utf-8"` too.
|
|
62
|
+
The same holds for `subprocess.run()` with `text=True`, which ruff does not check either.
|
|
46
63
|
|
|
47
64
|
### Docstrings
|
|
48
65
|
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: stepup-queue
|
|
3
|
-
Version: 2.0
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 2.1.0
|
|
4
|
+
Summary: Integrate SLURM jobs into a StepUp workflow
|
|
5
5
|
Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
|
|
6
6
|
License-Expression: LGPL-3.0-or-later
|
|
7
7
|
Project-URL: Documentation, https://reproducible-reporting.github.io/stepup-queue/
|
|
8
8
|
Project-URL: Issues, https://github.com/reproducible-reporting/stepup-queue/issues
|
|
9
9
|
Project-URL: Source, https://github.com/reproducible-reporting/stepup-queue/
|
|
10
10
|
Project-URL: Changelog, https://reproducible-reporting.github.io/stepup-queue/changelog/
|
|
11
|
+
Keywords: asyncio,batch computing,build automation,build tool,extension,high-performance computing,HPC,job scheduler,job submission,parallel,POSIX,Python,queued jobs,reproducibility,reproducible research,research software,SLURM,StepUp,StepUp Core,supercomputer,workflow automation,workflow management
|
|
11
12
|
Classifier: Development Status :: 4 - Beta
|
|
12
13
|
Classifier: Environment :: Console
|
|
13
14
|
Classifier: Intended Audience :: Education
|
|
@@ -20,25 +21,24 @@ Classifier: Programming Language :: Python :: 3.11
|
|
|
20
21
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
22
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
23
|
Classifier: Programming Language :: Python :: 3.14
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.15
|
|
23
25
|
Classifier: Topic :: Software Development :: Build Tools
|
|
24
26
|
Requires-Python: >=3.11
|
|
25
27
|
Description-Content-Type: text/markdown
|
|
26
28
|
License-File: LICENSE
|
|
27
|
-
Requires-Dist: path>=16.14.0
|
|
28
29
|
Requires-Dist: rich>=13.0.0
|
|
29
|
-
Requires-Dist: stepup<5.0.0a1,>=4.0
|
|
30
|
+
Requires-Dist: stepup<5.0.0a1,>=4.1.0
|
|
31
|
+
Provides-Extra: tests
|
|
32
|
+
Requires-Dist: pytest>=8.2.0; extra == "tests"
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
|
|
34
|
+
Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
|
|
35
|
+
Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
|
|
30
36
|
Provides-Extra: dev
|
|
31
|
-
Requires-Dist:
|
|
32
|
-
Requires-Dist:
|
|
33
|
-
Requires-Dist: mkdocs; extra == "dev"
|
|
34
|
-
Requires-Dist: mkdocstrings[python]; extra == "dev"
|
|
35
|
-
Requires-Dist: mkdocs-macros-plugin; extra == "dev"
|
|
36
|
-
Requires-Dist: mkdocs-material; extra == "dev"
|
|
37
|
-
Requires-Dist: psutil; extra == "dev"
|
|
38
|
-
Requires-Dist: pytest; extra == "dev"
|
|
39
|
-
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
40
|
-
Requires-Dist: pytest-xdist; extra == "dev"
|
|
37
|
+
Requires-Dist: stepup-queue[tests]; extra == "dev"
|
|
38
|
+
Requires-Dist: mkdocstrings-python; extra == "dev"
|
|
41
39
|
Requires-Dist: reuse; extra == "dev"
|
|
40
|
+
Requires-Dist: snipwise<2.0a0; extra == "dev"
|
|
41
|
+
Requires-Dist: zensical; extra == "dev"
|
|
42
42
|
Dynamic: license-file
|
|
43
43
|
|
|
44
44
|
<!--
|
|
@@ -50,15 +50,17 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|
|
50
50
|
|
|
51
51
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml)
|
|
52
52
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
|
|
53
|
-
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml)
|
|
54
54
|
[](https://pypi.org/project/stepup-queue/)
|
|
55
55
|
[](https://doi.org/10.5281/zenodo.22024209)
|
|
56
56
|

|
|
57
57
|

|
|
58
58
|
|
|
59
|
+
<!-- snipwise.md BEGIN abstract -->
|
|
59
60
|
StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
|
|
60
61
|
extension to integrate queued jobs into a workflow.
|
|
61
|
-
|
|
62
|
-
|
|
62
|
+
It currently supports integration with [SLURM](https://slurm.schedmd.com/)
|
|
63
|
+
and is designed to be extensible to other job schedulers.
|
|
64
|
+
<!-- snipwise.md END abstract -->
|
|
63
65
|
|
|
64
66
|
For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).
|
|
@@ -7,15 +7,17 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|
|
7
7
|
|
|
8
8
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml)
|
|
9
9
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
|
|
10
|
-
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml)
|
|
11
11
|
[](https://pypi.org/project/stepup-queue/)
|
|
12
12
|
[](https://doi.org/10.5281/zenodo.22024209)
|
|
13
13
|

|
|
14
14
|

|
|
15
15
|
|
|
16
|
+
<!-- snipwise.md BEGIN abstract -->
|
|
16
17
|
StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
|
|
17
18
|
extension to integrate queued jobs into a workflow.
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
It currently supports integration with [SLURM](https://slurm.schedmd.com/)
|
|
20
|
+
and is designed to be extensible to other job schedulers.
|
|
21
|
+
<!-- snipwise.md END abstract -->
|
|
20
22
|
|
|
21
23
|
For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).
|
|
@@ -15,11 +15,10 @@ SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
|
15
15
|
SPDX-License-Identifier = "LGPL-3.0-or-later"
|
|
16
16
|
|
|
17
17
|
[[annotations]]
|
|
18
|
-
path =
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
path = "docs/logo.svg"
|
|
18
|
+
path = [
|
|
19
|
+
"docs/logo.svg",
|
|
20
|
+
"docs/social-card.jpg",
|
|
21
|
+
"docs/social-card.svg",
|
|
22
|
+
]
|
|
24
23
|
SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
25
24
|
SPDX-License-Identifier = "LicenseRef-StepUp-Logo-v1"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
2
2
|
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
3
3
|
[build-system]
|
|
4
|
-
requires = ["setuptools>=
|
|
4
|
+
requires = ["setuptools>=77.0", "setuptools_scm>=8.1"]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
6
6
|
|
|
7
7
|
[project]
|
|
@@ -9,10 +9,36 @@ name = "stepup-queue"
|
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="Toon Verstraelen", email="toon.verstraelen@ugent.be" },
|
|
11
11
|
]
|
|
12
|
-
description = "
|
|
12
|
+
description = "Integrate SLURM jobs into a StepUp workflow"
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
license = "LGPL-3.0-or-later"
|
|
15
15
|
requires-python = ">=3.11"
|
|
16
|
+
keywords = [
|
|
17
|
+
# snipwise.md BEGIN keywords
|
|
18
|
+
"asyncio",
|
|
19
|
+
"batch computing",
|
|
20
|
+
"build automation",
|
|
21
|
+
"build tool",
|
|
22
|
+
"extension",
|
|
23
|
+
"high-performance computing",
|
|
24
|
+
"HPC",
|
|
25
|
+
"job scheduler",
|
|
26
|
+
"job submission",
|
|
27
|
+
"parallel",
|
|
28
|
+
"POSIX",
|
|
29
|
+
"Python",
|
|
30
|
+
"queued jobs",
|
|
31
|
+
"reproducibility",
|
|
32
|
+
"reproducible research",
|
|
33
|
+
"research software",
|
|
34
|
+
"SLURM",
|
|
35
|
+
"StepUp",
|
|
36
|
+
"StepUp Core",
|
|
37
|
+
"supercomputer",
|
|
38
|
+
"workflow automation",
|
|
39
|
+
"workflow management",
|
|
40
|
+
# snipwise.md END keywords
|
|
41
|
+
]
|
|
16
42
|
classifiers = [
|
|
17
43
|
"Development Status :: 4 - Beta",
|
|
18
44
|
"Environment :: Console",
|
|
@@ -26,29 +52,32 @@ classifiers = [
|
|
|
26
52
|
"Programming Language :: Python :: 3.12",
|
|
27
53
|
"Programming Language :: Python :: 3.13",
|
|
28
54
|
"Programming Language :: Python :: 3.14",
|
|
55
|
+
"Programming Language :: Python :: 3.15",
|
|
29
56
|
"Topic :: Software Development :: Build Tools",
|
|
30
57
|
]
|
|
31
58
|
dependencies = [
|
|
32
|
-
# Ensure changes to these dependencies are reflected in .github/requirements-old.txt
|
|
33
|
-
"path>=16.14.0",
|
|
34
59
|
"rich>=13.0.0",
|
|
35
|
-
"stepup>=4.0
|
|
60
|
+
"stepup>=4.1.0,<5.0.0a1",
|
|
36
61
|
]
|
|
37
62
|
dynamic = ["version"]
|
|
38
63
|
|
|
39
64
|
[project.optional-dependencies]
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
"
|
|
46
|
-
"mkdocs-material",
|
|
47
|
-
"psutil",
|
|
48
|
-
"pytest",
|
|
65
|
+
# What the test suite needs, and nothing more.
|
|
66
|
+
# The pytest workflow installs this extra rather than `dev`,
|
|
67
|
+
# because it resolves the oldest supported versions of the direct dependencies,
|
|
68
|
+
# which is only meaningful for packages that the tests actually exercise.
|
|
69
|
+
tests = [
|
|
70
|
+
"pytest>=8.2.0",
|
|
49
71
|
"pytest-asyncio>=0.24",
|
|
50
|
-
"pytest-
|
|
72
|
+
"pytest-timeout>=2.3.1",
|
|
73
|
+
"pytest-xdist>=3.6.1",
|
|
74
|
+
]
|
|
75
|
+
dev = [
|
|
76
|
+
"stepup-queue[tests]",
|
|
77
|
+
"mkdocstrings-python",
|
|
51
78
|
"reuse",
|
|
79
|
+
"snipwise<2.0a0",
|
|
80
|
+
"zensical",
|
|
52
81
|
]
|
|
53
82
|
|
|
54
83
|
[project.urls]
|
|
@@ -58,24 +87,28 @@ Source = "https://github.com/reproducible-reporting/stepup-queue/"
|
|
|
58
87
|
Changelog = "https://reproducible-reporting.github.io/stepup-queue/changelog/"
|
|
59
88
|
|
|
60
89
|
[project.scripts]
|
|
90
|
+
sq-cancel-jobs = "stepup.queue.cancel_jobs:cancel_jobs"
|
|
91
|
+
sq-remove-jobs = "stepup.queue.remove_jobs:remove_jobs"
|
|
61
92
|
sq-sbatch-and-wait = "stepup.queue.sbatch:sbatch"
|
|
62
93
|
|
|
63
|
-
[project.entry-points."stepup.tools"]
|
|
64
|
-
canceljobs = "stepup.queue.canceljobs:canceljobs_subcommand"
|
|
65
|
-
removejobs = "stepup.queue.removejobs:removejobs_subcommand"
|
|
66
|
-
|
|
67
94
|
[tool.pytest.ini_options]
|
|
68
95
|
addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
|
|
69
96
|
asyncio_default_fixture_loop_scope = "function"
|
|
97
|
+
timeout = 15
|
|
70
98
|
|
|
71
99
|
[tool.ruff]
|
|
72
100
|
line-length = 100
|
|
73
101
|
target-version = "py311"
|
|
74
102
|
|
|
75
103
|
[tool.ruff.lint]
|
|
104
|
+
# Preview mode is only enabled for the preview rules selected by their full code below,
|
|
105
|
+
# but it also turns on the preview behavior of stable rules.
|
|
106
|
+
preview = true
|
|
107
|
+
explicit-preview-rules = true
|
|
76
108
|
select = [
|
|
77
109
|
"A", "B", "BLE", "C4", "E", "EXE", "F", "I", "ICN", "ISC", "N", "NPY", "PERF", "PIE", "PL",
|
|
78
|
-
"PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W"
|
|
110
|
+
"PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W",
|
|
111
|
+
"PLW1514", # https://docs.astral.sh/ruff/rules/unspecified-encoding/ (preview in ruff 0.17.2)
|
|
79
112
|
]
|
|
80
113
|
ignore = [
|
|
81
114
|
"PLR0904", # https://docs.astral.sh/ruff/rules/too-many-public-methods/
|
|
@@ -109,7 +142,10 @@ local_scheme = "no-local-version"
|
|
|
109
142
|
[tool.pymarkdown]
|
|
110
143
|
# Report unknown or mistyped settings instead of silently ignoring them.
|
|
111
144
|
mode.strict-config = true
|
|
112
|
-
# The documentation
|
|
145
|
+
# The documentation pages carry YAML front matter,
|
|
146
|
+
# which pymarkdown parses as a setext heading unless this extension is on.
|
|
147
|
+
extensions.front-matter.enabled = true
|
|
148
|
+
# The documentation indents nested lists by four spaces, matching Python-Markdown and Zensical.
|
|
113
149
|
plugins.md007.indent = 4
|
|
114
150
|
plugins.md013.line_length = 100
|
|
115
151
|
plugins.md013.heading_line_length = 100
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
|
+
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
|
+
-->
|
|
5
|
+
|
|
6
|
+
# Snipwise Configuration
|
|
7
|
+
|
|
8
|
+
Consult the full documentation at <https://reproducible-reporting.github.io/snipwise/>.
|
|
9
|
+
|
|
10
|
+
```toml
|
|
11
|
+
# The Markdown files show the abstract as it is written below, links and all.
|
|
12
|
+
[[targets]]
|
|
13
|
+
patterns = ["README.md", "docs/index.md"]
|
|
14
|
+
|
|
15
|
+
# The summary of the Python package metadata, which is a single-line TOML string.
|
|
16
|
+
[[targets]]
|
|
17
|
+
patterns = ["pyproject.toml"]
|
|
18
|
+
scanner = "regex"
|
|
19
|
+
regex = '(?m)^description = "(?P<content>[^"]*)"$'
|
|
20
|
+
snippets = ["tagline"]
|
|
21
|
+
render = "{{ content | unwrap }}"
|
|
22
|
+
|
|
23
|
+
# The meta description that every documentation page carries.
|
|
24
|
+
[[targets]]
|
|
25
|
+
patterns = ["zensical.toml"]
|
|
26
|
+
scanner = "regex"
|
|
27
|
+
regex = '(?m)^site_description = "(?P<content>[^"]*)"$'
|
|
28
|
+
snippets = ["tagline"]
|
|
29
|
+
render = "{{ content | unwrap }}"
|
|
30
|
+
|
|
31
|
+
# The docstring of the package, which is a single-line string.
|
|
32
|
+
[[targets]]
|
|
33
|
+
patterns = ["stepup/queue/__init__.py"]
|
|
34
|
+
scanner = "regex"
|
|
35
|
+
regex = '(?m)^"""(?P<content>[^"]*)"""$'
|
|
36
|
+
snippets = ["tagline"]
|
|
37
|
+
render = "{{ content | unwrap | suffix('.') }}"
|
|
38
|
+
|
|
39
|
+
# The tagline in the social card, whose tspan carries the id that locates it.
|
|
40
|
+
[[targets]]
|
|
41
|
+
patterns = ["docs/social-card.svg"]
|
|
42
|
+
scanner = "regex"
|
|
43
|
+
regex = '<tspan\b[^>]*\bid="tagline"[^>]*>(?P<content>[^<]*)</tspan>'
|
|
44
|
+
snippets = ["tagline"]
|
|
45
|
+
render = "{{ content | unwrap }}"
|
|
46
|
+
|
|
47
|
+
# The keywords array of the Python package metadata.
|
|
48
|
+
[[targets]]
|
|
49
|
+
patterns = ["pyproject.toml"]
|
|
50
|
+
snippets = ["keywords"]
|
|
51
|
+
render = '''{{ content | prefix('"') | suffix('",') }}'''
|
|
52
|
+
|
|
53
|
+
# The abstract of the citation metadata, which is a folded YAML block scalar.
|
|
54
|
+
# The template terminates the last line, because the region includes its newline.
|
|
55
|
+
[[targets]]
|
|
56
|
+
patterns = ["CITATION.cff"]
|
|
57
|
+
scanner = "regex"
|
|
58
|
+
regex = '(?m)^abstract: >-\n(?P<content>(?:^ .*\n)+)'
|
|
59
|
+
snippets = ["abstract"]
|
|
60
|
+
render = "{{ content | plain | prefix(' ') }}\n"
|
|
61
|
+
|
|
62
|
+
# The keywords sequence of the citation metadata.
|
|
63
|
+
[[targets]]
|
|
64
|
+
patterns = ["CITATION.cff"]
|
|
65
|
+
snippets = ["keywords"]
|
|
66
|
+
render = "{{ content | prefix('- ') }}"
|
|
67
|
+
|
|
68
|
+
# The Zenodo metadata, which is JSON and therefore carries no markers.
|
|
69
|
+
[[targets]]
|
|
70
|
+
patterns = [".zenodo.json"]
|
|
71
|
+
scanner = "json"
|
|
72
|
+
insert = [
|
|
73
|
+
{ snippet = "abstract", pointer = "/description", render = "{{ content | plain | unwrap }}" },
|
|
74
|
+
{ snippet = "keywords", pointer = "/keywords", shape = "lines" },
|
|
75
|
+
]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## `tagline`
|
|
79
|
+
|
|
80
|
+
```text
|
|
81
|
+
Integrate SLURM jobs into a StepUp workflow
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## `abstract`
|
|
85
|
+
|
|
86
|
+
```markdown
|
|
87
|
+
StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
|
|
88
|
+
extension to integrate queued jobs into a workflow.
|
|
89
|
+
It currently supports integration with [SLURM](https://slurm.schedmd.com/)
|
|
90
|
+
and is designed to be extensible to other job schedulers.
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## `keywords`
|
|
94
|
+
|
|
95
|
+
```text
|
|
96
|
+
asyncio
|
|
97
|
+
batch computing
|
|
98
|
+
build automation
|
|
99
|
+
build tool
|
|
100
|
+
extension
|
|
101
|
+
high-performance computing
|
|
102
|
+
HPC
|
|
103
|
+
job scheduler
|
|
104
|
+
job submission
|
|
105
|
+
parallel
|
|
106
|
+
POSIX
|
|
107
|
+
Python
|
|
108
|
+
queued jobs
|
|
109
|
+
reproducibility
|
|
110
|
+
reproducible research
|
|
111
|
+
research software
|
|
112
|
+
SLURM
|
|
113
|
+
StepUp
|
|
114
|
+
StepUp Core
|
|
115
|
+
supercomputer
|
|
116
|
+
workflow automation
|
|
117
|
+
workflow management
|
|
118
|
+
```
|
|
@@ -31,10 +31,24 @@ attaches to the job it already submitted, instead of submitting a second one:
|
|
|
31
31
|
3. With a log whose digest matches, it resumes waiting for the existing job.
|
|
32
32
|
4. With a digest mismatch, the `onchange` policy decides
|
|
33
33
|
between `raise`, `resubmit` and `ignore`.
|
|
34
|
+
With `raise` or `resubmit`, an optional `complete` command is consulted first.
|
|
35
|
+
When it succeeds, the new digest is written into the log and the existing job is kept,
|
|
36
|
+
which brings the step back to case 3.
|
|
34
37
|
|
|
35
38
|
This is why the digest is written into the log rather than derived at read time:
|
|
36
39
|
the log has to be interpretable by a process that never saw the original submission.
|
|
37
40
|
|
|
41
|
+
The invariant behind the contract is that at most one job runs per job directory,
|
|
42
|
+
because every job in it writes to the same `slurmjob.out`, `slurmjob.err` and `slurmjob.ret`.
|
|
43
|
+
Two places protect it:
|
|
44
|
+
|
|
45
|
+
- The `Submitting` marker is logged before `sbatch` runs and replaced by `Submitted <jobid>`
|
|
46
|
+
once the job ID is known.
|
|
47
|
+
A log that stops at the marker means the job ID may have been lost,
|
|
48
|
+
so the next run refuses to submit rather than risk a second job next to an untracked one.
|
|
49
|
+
- `onchange="resubmit"` waits for the cancelled job to leave the queue before resubmitting,
|
|
50
|
+
because `scancel` returns as soon as the request is accepted.
|
|
51
|
+
|
|
38
52
|
## sacct Caching
|
|
39
53
|
|
|
40
54
|
All concurrent `sq-sbatch-and-wait` processes share one cached `sacct` result per cluster,
|
|
@@ -21,6 +21,7 @@ def sbatch(
|
|
|
21
21
|
out: Collection[StrPath] | StrPath = (),
|
|
22
22
|
vol: Collection[StrPath] | StrPath = (),
|
|
23
23
|
onchange: str | None = None,
|
|
24
|
+
complete: str | None = None,
|
|
24
25
|
optional: bool = False,
|
|
25
26
|
resources: dict[str, int] | str | None = None,
|
|
26
27
|
duration: float | None = None,
|
|
@@ -63,8 +64,23 @@ def sbatch(
|
|
|
63
64
|
and pass it here as `source rc.sh`.
|
|
64
65
|
In this case, you usually also want to include `rc.sh` in the `inp` list.
|
|
65
66
|
onchange
|
|
66
|
-
Policy when
|
|
67
|
+
Policy when the inputs of a previously submitted job have changed.
|
|
67
68
|
Must be one of `"raise"`, `"resubmit"` or `"ignore"`.
|
|
69
|
+
When not given, the environment variable `STEPUP_QUEUE_ONCHANGE` is used,
|
|
70
|
+
with `"raise"` as default.
|
|
71
|
+
complete
|
|
72
|
+
A shell command that tells whether the outputs of the job are complete,
|
|
73
|
+
e.g. `"test -f result.txt"`.
|
|
74
|
+
The command is executed in the working directory
|
|
75
|
+
when the inputs of a previously submitted job have changed.
|
|
76
|
+
A zero exit code keeps the existing job, as if its inputs had not changed.
|
|
77
|
+
A non-zero exit code applies the `onchange` policy.
|
|
78
|
+
Hence, with `onchange="resubmit"`, a job whose inputs have changed is only resubmitted
|
|
79
|
+
when outputs are missing.
|
|
80
|
+
This is useful for job scripts that skip work whose outputs already exist.
|
|
81
|
+
With `onchange="raise"`, an error is only raised when outputs are missing.
|
|
82
|
+
The command is not used with `onchange="ignore"`,
|
|
83
|
+
because that policy keeps the existing job anyway.
|
|
68
84
|
"""
|
|
69
85
|
if ext == "":
|
|
70
86
|
ext = ".sh"
|
|
@@ -81,6 +97,10 @@ def sbatch(
|
|
|
81
97
|
if onchange not in ["raise", "resubmit", "ignore"]:
|
|
82
98
|
raise ValueError(f"Invalid onchange policy {onchange}.")
|
|
83
99
|
cmd += f" --onchange={onchange}"
|
|
100
|
+
if complete is not None:
|
|
101
|
+
if complete.strip() == "":
|
|
102
|
+
raise ValueError("The complete command must not be empty.")
|
|
103
|
+
cmd += f" --complete={shlex.quote(complete)}"
|
|
84
104
|
return run(
|
|
85
105
|
cmd,
|
|
86
106
|
inp=[f"slurmjob{ext}", *coerce_paths(inp)],
|