stepup-queue 2.0.0rc3__tar.gz → 2.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepup_queue-2.0.1/.zenodo.json +56 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/CITATION.cff +38 -2
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/CLAUDE.md +7 -4
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/MANIFEST.in +0 -1
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/PKG-INFO +16 -12
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/README.md +5 -3
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/REUSE.toml +10 -1
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/pyproject.toml +62 -14
- stepup_queue-2.0.1/snipwise.md +118 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup/queue/CLAUDE.md +12 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup/queue/__init__.py +1 -1
- stepup_queue-2.0.0rc3/stepup/queue/canceljobs.py → stepup_queue-2.0.1/stepup/queue/cancel_jobs.py +19 -18
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup/queue/log.py +40 -8
- stepup_queue-2.0.0rc3/stepup/queue/removejobs.py → stepup_queue-2.0.1/stepup/queue/remove_jobs.py +33 -14
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup/queue/sbatch.py +150 -44
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup/queue/utils.py +3 -2
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup_queue.egg-info/PKG-INFO +16 -12
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup_queue.egg-info/SOURCES.txt +4 -2
- stepup_queue-2.0.1/stepup_queue.egg-info/entry_points.txt +4 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup_queue.egg-info/requires.txt +9 -7
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup_queue.egg-info/scm_file_list.json +11 -5
- stepup_queue-2.0.1/stepup_queue.egg-info/scm_version.json +8 -0
- stepup_queue-2.0.0rc3/stepup_queue.egg-info/entry_points.txt +0 -6
- stepup_queue-2.0.0rc3/stepup_queue.egg-info/scm_version.json +0 -8
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/LICENSE +0 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/LICENSES/CC-BY-SA-4.0.txt +0 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/LICENSES/LGPL-3.0-or-later.txt +0 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/setup.cfg +0 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup/queue/api.py +0 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup_queue.egg-info/dependency_links.txt +0 -0
- {stepup_queue-2.0.0rc3 → stepup_queue-2.0.1}/stepup_queue.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"upload_type": "software",
|
|
3
|
+
"title": "StepUp Queue",
|
|
4
|
+
"description": "StepUp Queue is an experimental StepUp extension to integrate queued jobs into a workflow. It currently supports integration with SLURM and is designed to be extensible to other job schedulers.",
|
|
5
|
+
"creators": [
|
|
6
|
+
{
|
|
7
|
+
"name": "Verstraelen, Toon",
|
|
8
|
+
"orcid": "0000-0001-9288-5608",
|
|
9
|
+
"affiliation": "Center for Molecular Modeling (CMM), Ghent University"
|
|
10
|
+
}
|
|
11
|
+
],
|
|
12
|
+
"license": "lgpl-3.0-or-later",
|
|
13
|
+
"access_right": "open",
|
|
14
|
+
"communities": [
|
|
15
|
+
{
|
|
16
|
+
"identifier": "ugent"
|
|
17
|
+
}
|
|
18
|
+
],
|
|
19
|
+
"language": "eng",
|
|
20
|
+
"keywords": [
|
|
21
|
+
"asyncio",
|
|
22
|
+
"batch computing",
|
|
23
|
+
"build automation",
|
|
24
|
+
"build tool",
|
|
25
|
+
"extension",
|
|
26
|
+
"high-performance computing",
|
|
27
|
+
"HPC",
|
|
28
|
+
"job scheduler",
|
|
29
|
+
"job submission",
|
|
30
|
+
"parallel",
|
|
31
|
+
"POSIX",
|
|
32
|
+
"Python",
|
|
33
|
+
"queued jobs",
|
|
34
|
+
"reproducibility",
|
|
35
|
+
"reproducible research",
|
|
36
|
+
"research software",
|
|
37
|
+
"SLURM",
|
|
38
|
+
"StepUp",
|
|
39
|
+
"StepUp Core",
|
|
40
|
+
"supercomputer",
|
|
41
|
+
"workflow automation",
|
|
42
|
+
"workflow management"
|
|
43
|
+
],
|
|
44
|
+
"related_identifiers": [
|
|
45
|
+
{
|
|
46
|
+
"identifier": "https://reproducible-reporting.github.io/stepup-queue",
|
|
47
|
+
"relation": "isDocumentedBy",
|
|
48
|
+
"resource_type": "publication-softwaredocumentation"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"identifier": "10.5281/zenodo.22018869",
|
|
52
|
+
"relation": "requires",
|
|
53
|
+
"resource_type": "software"
|
|
54
|
+
}
|
|
55
|
+
]
|
|
56
|
+
}
|
|
@@ -6,7 +6,10 @@ title: StepUp Queue
|
|
|
6
6
|
type: software
|
|
7
7
|
url: https://reproducible-reporting.github.io/stepup-queue
|
|
8
8
|
abstract: >-
|
|
9
|
-
StepUp Queue is
|
|
9
|
+
StepUp Queue is an experimental StepUp
|
|
10
|
+
extension to integrate queued jobs into a workflow.
|
|
11
|
+
It currently supports integration with SLURM
|
|
12
|
+
and is designed to be extensible to other job schedulers.
|
|
10
13
|
authors:
|
|
11
14
|
- family-names: Verstraelen
|
|
12
15
|
given-names: Toon
|
|
@@ -18,6 +21,35 @@ authors:
|
|
|
18
21
|
country: BE
|
|
19
22
|
license: LGPL-3.0-or-later
|
|
20
23
|
repository-code: https://github.com/reproducible-reporting/stepup-queue
|
|
24
|
+
keywords:
|
|
25
|
+
# snipwise.md BEGIN keywords
|
|
26
|
+
- asyncio
|
|
27
|
+
- batch computing
|
|
28
|
+
- build automation
|
|
29
|
+
- build tool
|
|
30
|
+
- extension
|
|
31
|
+
- high-performance computing
|
|
32
|
+
- HPC
|
|
33
|
+
- job scheduler
|
|
34
|
+
- job submission
|
|
35
|
+
- parallel
|
|
36
|
+
- POSIX
|
|
37
|
+
- Python
|
|
38
|
+
- queued jobs
|
|
39
|
+
- reproducibility
|
|
40
|
+
- reproducible research
|
|
41
|
+
- research software
|
|
42
|
+
- SLURM
|
|
43
|
+
- StepUp
|
|
44
|
+
- StepUp Core
|
|
45
|
+
- supercomputer
|
|
46
|
+
- workflow automation
|
|
47
|
+
- workflow management
|
|
48
|
+
# snipwise.md END keywords
|
|
49
|
+
identifiers:
|
|
50
|
+
- type: doi
|
|
51
|
+
value: 10.5281/zenodo.22024209
|
|
52
|
+
description: Concept DOI, which always resolves to the latest archived version.
|
|
21
53
|
references:
|
|
22
54
|
- title: StepUp Core
|
|
23
55
|
type: software
|
|
@@ -29,4 +61,8 @@ references:
|
|
|
29
61
|
url: https://reproducible-reporting.github.io/stepup-core
|
|
30
62
|
repository-code: https://github.com/reproducible-reporting/stepup-core
|
|
31
63
|
license: LGPL-3.0-or-later
|
|
32
|
-
|
|
64
|
+
identifiers:
|
|
65
|
+
- type: doi
|
|
66
|
+
value: 10.5281/zenodo.22018869
|
|
67
|
+
description: Concept DOI, which always resolves to the latest archived version.
|
|
68
|
+
notes: StepUp Queue is an extension of StepUp Core and requires it at runtime.
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
3
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
4
|
-->
|
|
5
|
+
|
|
5
6
|
# CLAUDE.md
|
|
6
7
|
|
|
7
8
|
This file provides guidance to Claude Code (claude.ai/code)
|
|
@@ -11,6 +12,7 @@ Guidance that applies to only part of the repo lives next to the code it governs
|
|
|
11
12
|
|
|
12
13
|
- `stepup/queue/CLAUDE.md`: job file conventions, the submit-and-wait contract, sacct caching.
|
|
13
14
|
- `tests/CLAUDE.md`: test layout and conventions.
|
|
15
|
+
- `docs/CLAUDE.md`: regenerating the social card.
|
|
14
16
|
|
|
15
17
|
## Project Overview
|
|
16
18
|
|
|
@@ -30,10 +32,11 @@ The package is organized around two layers:
|
|
|
30
32
|
Its `sbatch()` function calls `stepup.core.api.run()`
|
|
31
33
|
to register a `sq-sbatch-and-wait` step with StepUp Core.
|
|
32
34
|
|
|
33
|
-
- **Individual command modules** each implement a
|
|
34
|
-
These are registered via `[project.scripts]`
|
|
35
|
-
|
|
36
|
-
|
|
35
|
+
- **Individual command modules** each implement a function that serves as a CLI entry point.
|
|
36
|
+
These are registered via `[project.scripts]` in `pyproject.toml`.
|
|
37
|
+
`sq-sbatch-and-wait` is invoked by StepUp steps as an external command,
|
|
38
|
+
and runs in the working directory of the job.
|
|
39
|
+
`sq-cancel-jobs` and `sq-remove-jobs` are run by the user from the shell.
|
|
37
40
|
|
|
38
41
|
## Coding Conventions
|
|
39
42
|
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: stepup-queue
|
|
3
|
-
Version: 2.0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 2.0.1
|
|
4
|
+
Summary: Integrate SLURM jobs into a StepUp workflow
|
|
5
5
|
Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
|
|
6
6
|
License-Expression: LGPL-3.0-or-later
|
|
7
7
|
Project-URL: Documentation, https://reproducible-reporting.github.io/stepup-queue/
|
|
8
8
|
Project-URL: Issues, https://github.com/reproducible-reporting/stepup-queue/issues
|
|
9
9
|
Project-URL: Source, https://github.com/reproducible-reporting/stepup-queue/
|
|
10
10
|
Project-URL: Changelog, https://reproducible-reporting.github.io/stepup-queue/changelog/
|
|
11
|
+
Keywords: asyncio,batch computing,build automation,build tool,extension,high-performance computing,HPC,job scheduler,job submission,parallel,POSIX,Python,queued jobs,reproducibility,reproducible research,research software,SLURM,StepUp,StepUp Core,supercomputer,workflow automation,workflow management
|
|
11
12
|
Classifier: Development Status :: 4 - Beta
|
|
12
13
|
Classifier: Environment :: Console
|
|
13
14
|
Classifier: Intended Audience :: Education
|
|
@@ -26,26 +27,26 @@ Description-Content-Type: text/markdown
|
|
|
26
27
|
License-File: LICENSE
|
|
27
28
|
Requires-Dist: path>=16.14.0
|
|
28
29
|
Requires-Dist: rich>=13.0.0
|
|
29
|
-
Requires-Dist: stepup<5.0.0a1,>=4.0.
|
|
30
|
+
Requires-Dist: stepup<5.0.0a1,>=4.0.3
|
|
31
|
+
Provides-Extra: tests
|
|
32
|
+
Requires-Dist: pytest>=8.2.0; extra == "tests"
|
|
33
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
|
|
34
|
+
Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
|
|
35
|
+
Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
|
|
30
36
|
Provides-Extra: dev
|
|
31
|
-
Requires-Dist:
|
|
32
|
-
Requires-Dist: mike; extra == "dev"
|
|
37
|
+
Requires-Dist: stepup-queue[tests]; extra == "dev"
|
|
33
38
|
Requires-Dist: mkdocs; extra == "dev"
|
|
34
39
|
Requires-Dist: mkdocstrings[python]; extra == "dev"
|
|
35
40
|
Requires-Dist: mkdocs-macros-plugin; extra == "dev"
|
|
36
41
|
Requires-Dist: mkdocs-material; extra == "dev"
|
|
37
|
-
Requires-Dist: psutil; extra == "dev"
|
|
38
|
-
Requires-Dist: pytest; extra == "dev"
|
|
39
|
-
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
40
|
-
Requires-Dist: pytest-xdist; extra == "dev"
|
|
41
42
|
Requires-Dist: reuse; extra == "dev"
|
|
43
|
+
Requires-Dist: snipwise<2.0a0; extra == "dev"
|
|
42
44
|
Dynamic: license-file
|
|
43
45
|
|
|
44
46
|
<!--
|
|
45
47
|
SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
46
48
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
47
49
|
-->
|
|
48
|
-
<!-- markdownlint-disable line-length -->
|
|
49
50
|
|
|
50
51
|
# StepUp Queue
|
|
51
52
|
|
|
@@ -53,12 +54,15 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|
|
53
54
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
|
|
54
55
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
|
|
55
56
|
[](https://pypi.org/project/stepup-queue/)
|
|
57
|
+
[](https://doi.org/10.5281/zenodo.22024209)
|
|
56
58
|

|
|
57
59
|

|
|
58
60
|
|
|
61
|
+
<!-- snipwise.md BEGIN abstract -->
|
|
59
62
|
StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
|
|
60
63
|
extension to integrate queued jobs into a workflow.
|
|
61
|
-
|
|
62
|
-
|
|
64
|
+
It currently supports integration with [SLURM](https://slurm.schedmd.com/)
|
|
65
|
+
and is designed to be extensible to other job schedulers.
|
|
66
|
+
<!-- snipwise.md END abstract -->
|
|
63
67
|
|
|
64
68
|
For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
3
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
4
|
-->
|
|
5
|
-
<!-- markdownlint-disable line-length -->
|
|
6
5
|
|
|
7
6
|
# StepUp Queue
|
|
8
7
|
|
|
@@ -10,12 +9,15 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|
|
10
9
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
|
|
11
10
|
[](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
|
|
12
11
|
[](https://pypi.org/project/stepup-queue/)
|
|
12
|
+
[](https://doi.org/10.5281/zenodo.22024209)
|
|
13
13
|

|
|
14
14
|

|
|
15
15
|
|
|
16
|
+
<!-- snipwise.md BEGIN abstract -->
|
|
16
17
|
StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
|
|
17
18
|
extension to integrate queued jobs into a workflow.
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
It currently supports integration with [SLURM](https://slurm.schedmd.com/)
|
|
20
|
+
and is designed to be extensible to other job schedulers.
|
|
21
|
+
<!-- snipwise.md END abstract -->
|
|
20
22
|
|
|
21
23
|
For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).
|
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
3
3
|
version = 1
|
|
4
4
|
|
|
5
|
+
[[annotations]]
|
|
6
|
+
path = ".zenodo.json"
|
|
7
|
+
SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
8
|
+
SPDX-License-Identifier = "LGPL-3.0-or-later"
|
|
9
|
+
|
|
5
10
|
[[annotations]]
|
|
6
11
|
path = [
|
|
7
12
|
"docs/examples/*/**/*",
|
|
@@ -15,6 +20,10 @@ SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
|
15
20
|
SPDX-License-Identifier = "LGPL-3.0-or-later"
|
|
16
21
|
|
|
17
22
|
[[annotations]]
|
|
18
|
-
path =
|
|
23
|
+
path = [
|
|
24
|
+
"docs/logo.svg",
|
|
25
|
+
"docs/social-card.jpg",
|
|
26
|
+
"docs/social-card.svg",
|
|
27
|
+
]
|
|
19
28
|
SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
20
29
|
SPDX-License-Identifier = "LicenseRef-StepUp-Logo-v1"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
2
2
|
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
3
3
|
[build-system]
|
|
4
|
-
requires = ["setuptools>=
|
|
4
|
+
requires = ["setuptools>=77.0", "setuptools_scm>=8.1"]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
6
6
|
|
|
7
7
|
[project]
|
|
@@ -9,10 +9,36 @@ name = "stepup-queue"
|
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="Toon Verstraelen", email="toon.verstraelen@ugent.be" },
|
|
11
11
|
]
|
|
12
|
-
description = "
|
|
12
|
+
description = "Integrate SLURM jobs into a StepUp workflow"
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
license = "LGPL-3.0-or-later"
|
|
15
15
|
requires-python = ">=3.11"
|
|
16
|
+
keywords = [
|
|
17
|
+
# snipwise.md BEGIN keywords
|
|
18
|
+
"asyncio",
|
|
19
|
+
"batch computing",
|
|
20
|
+
"build automation",
|
|
21
|
+
"build tool",
|
|
22
|
+
"extension",
|
|
23
|
+
"high-performance computing",
|
|
24
|
+
"HPC",
|
|
25
|
+
"job scheduler",
|
|
26
|
+
"job submission",
|
|
27
|
+
"parallel",
|
|
28
|
+
"POSIX",
|
|
29
|
+
"Python",
|
|
30
|
+
"queued jobs",
|
|
31
|
+
"reproducibility",
|
|
32
|
+
"reproducible research",
|
|
33
|
+
"research software",
|
|
34
|
+
"SLURM",
|
|
35
|
+
"StepUp",
|
|
36
|
+
"StepUp Core",
|
|
37
|
+
"supercomputer",
|
|
38
|
+
"workflow automation",
|
|
39
|
+
"workflow management",
|
|
40
|
+
# snipwise.md END keywords
|
|
41
|
+
]
|
|
16
42
|
classifiers = [
|
|
17
43
|
"Development Status :: 4 - Beta",
|
|
18
44
|
"Environment :: Console",
|
|
@@ -29,26 +55,31 @@ classifiers = [
|
|
|
29
55
|
"Topic :: Software Development :: Build Tools",
|
|
30
56
|
]
|
|
31
57
|
dependencies = [
|
|
32
|
-
# Ensure changes to these dependencies are reflected in .github/requirements-old.txt
|
|
33
58
|
"path>=16.14.0",
|
|
34
59
|
"rich>=13.0.0",
|
|
35
|
-
"stepup>=4.0.
|
|
60
|
+
"stepup>=4.0.3,<5.0.0a1",
|
|
36
61
|
]
|
|
37
62
|
dynamic = ["version"]
|
|
38
63
|
|
|
39
64
|
[project.optional-dependencies]
|
|
65
|
+
# What the test suite needs, and nothing more.
|
|
66
|
+
# The pytest workflow installs this extra rather than `dev`,
|
|
67
|
+
# because it resolves the oldest supported versions of the direct dependencies,
|
|
68
|
+
# which is only meaningful for packages that the tests actually exercise.
|
|
69
|
+
tests = [
|
|
70
|
+
"pytest>=8.2.0",
|
|
71
|
+
"pytest-asyncio>=0.24",
|
|
72
|
+
"pytest-timeout>=2.3.1",
|
|
73
|
+
"pytest-xdist>=3.6.1",
|
|
74
|
+
]
|
|
40
75
|
dev = [
|
|
41
|
-
"
|
|
42
|
-
"mike",
|
|
76
|
+
"stepup-queue[tests]",
|
|
43
77
|
"mkdocs",
|
|
44
78
|
"mkdocstrings[python]",
|
|
45
79
|
"mkdocs-macros-plugin",
|
|
46
80
|
"mkdocs-material",
|
|
47
|
-
"psutil",
|
|
48
|
-
"pytest",
|
|
49
|
-
"pytest-asyncio>=0.24",
|
|
50
|
-
"pytest-xdist",
|
|
51
81
|
"reuse",
|
|
82
|
+
"snipwise<2.0a0",
|
|
52
83
|
]
|
|
53
84
|
|
|
54
85
|
[project.urls]
|
|
@@ -58,15 +89,14 @@ Source = "https://github.com/reproducible-reporting/stepup-queue/"
|
|
|
58
89
|
Changelog = "https://reproducible-reporting.github.io/stepup-queue/changelog/"
|
|
59
90
|
|
|
60
91
|
[project.scripts]
|
|
92
|
+
sq-cancel-jobs = "stepup.queue.cancel_jobs:cancel_jobs"
|
|
93
|
+
sq-remove-jobs = "stepup.queue.remove_jobs:remove_jobs"
|
|
61
94
|
sq-sbatch-and-wait = "stepup.queue.sbatch:sbatch"
|
|
62
95
|
|
|
63
|
-
[project.entry-points."stepup.tools"]
|
|
64
|
-
canceljobs = "stepup.queue.canceljobs:canceljobs_subcommand"
|
|
65
|
-
removejobs = "stepup.queue.removejobs:removejobs_subcommand"
|
|
66
|
-
|
|
67
96
|
[tool.pytest.ini_options]
|
|
68
97
|
addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
|
|
69
98
|
asyncio_default_fixture_loop_scope = "function"
|
|
99
|
+
timeout = 15
|
|
70
100
|
|
|
71
101
|
[tool.ruff]
|
|
72
102
|
line-length = 100
|
|
@@ -105,3 +135,21 @@ packages = ["stepup.queue"]
|
|
|
105
135
|
[tool.setuptools_scm]
|
|
106
136
|
version_scheme = "post-release"
|
|
107
137
|
local_scheme = "no-local-version"
|
|
138
|
+
|
|
139
|
+
[tool.pymarkdown]
|
|
140
|
+
# Report unknown or mistyped settings instead of silently ignoring them.
|
|
141
|
+
mode.strict-config = true
|
|
142
|
+
# The documentation pages carry YAML front matter,
|
|
143
|
+
# which pymarkdown parses as a setext heading unless this extension is on.
|
|
144
|
+
extensions.front-matter.enabled = true
|
|
145
|
+
# The documentation indents nested lists by four spaces, matching Python-Markdown and MkDocs.
|
|
146
|
+
plugins.md007.indent = 4
|
|
147
|
+
plugins.md013.line_length = 100
|
|
148
|
+
plugins.md013.heading_line_length = 100
|
|
149
|
+
plugins.md013.code_block_line_length = 100
|
|
150
|
+
# The changelog repeats headings such as "Fixed" under every release,
|
|
151
|
+
# which is fine as long as they do not repeat under the same release.
|
|
152
|
+
plugins.md024.siblings_only = true
|
|
153
|
+
# Rule md041 (first-line-heading) counts the SPDX header in an HTML comment
|
|
154
|
+
# as content preceding the first heading.
|
|
155
|
+
plugins.md041.enabled = false
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
|
+
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
|
+
-->
|
|
5
|
+
|
|
6
|
+
# Snipwise Configuration
|
|
7
|
+
|
|
8
|
+
Consult the full documentation at <https://reproducible-reporting.github.io/snipwise/>.
|
|
9
|
+
|
|
10
|
+
```toml
|
|
11
|
+
# The Markdown files show the abstract as it is written below, links and all.
|
|
12
|
+
[[targets]]
|
|
13
|
+
patterns = ["README.md", "docs/index.md"]
|
|
14
|
+
|
|
15
|
+
# The summary of the Python package metadata, which is a single-line TOML string.
|
|
16
|
+
[[targets]]
|
|
17
|
+
patterns = ["pyproject.toml"]
|
|
18
|
+
scanner = "regex"
|
|
19
|
+
regex = '(?m)^description = "(?P<content>[^"]*)"$'
|
|
20
|
+
snippets = ["tagline"]
|
|
21
|
+
render = "{{ content | unwrap }}"
|
|
22
|
+
|
|
23
|
+
# The meta description that every documentation page carries.
|
|
24
|
+
[[targets]]
|
|
25
|
+
patterns = ["mkdocs.yaml"]
|
|
26
|
+
scanner = "regex"
|
|
27
|
+
regex = '(?m)^site_description: (?P<content>.*)$'
|
|
28
|
+
snippets = ["tagline"]
|
|
29
|
+
render = "{{ content | unwrap }}"
|
|
30
|
+
|
|
31
|
+
# The docstring of the package, which is a single-line string.
|
|
32
|
+
[[targets]]
|
|
33
|
+
patterns = ["stepup/queue/__init__.py"]
|
|
34
|
+
scanner = "regex"
|
|
35
|
+
regex = '(?m)^"""(?P<content>[^"]*)"""$'
|
|
36
|
+
snippets = ["tagline"]
|
|
37
|
+
render = "{{ content | unwrap | suffix('.') }}"
|
|
38
|
+
|
|
39
|
+
# The tagline in the social card, whose tspan carries the id that locates it.
|
|
40
|
+
[[targets]]
|
|
41
|
+
patterns = ["docs/social-card.svg"]
|
|
42
|
+
scanner = "regex"
|
|
43
|
+
regex = '<tspan\b[^>]*\bid="tagline"[^>]*>(?P<content>[^<]*)</tspan>'
|
|
44
|
+
snippets = ["tagline"]
|
|
45
|
+
render = "{{ content | unwrap }}"
|
|
46
|
+
|
|
47
|
+
# The keywords array of the Python package metadata.
|
|
48
|
+
[[targets]]
|
|
49
|
+
patterns = ["pyproject.toml"]
|
|
50
|
+
snippets = ["keywords"]
|
|
51
|
+
render = '''{{ content | prefix('"') | suffix('",') }}'''
|
|
52
|
+
|
|
53
|
+
# The abstract of the citation metadata, which is a folded YAML block scalar.
|
|
54
|
+
# The template terminates the last line, because the region includes its newline.
|
|
55
|
+
[[targets]]
|
|
56
|
+
patterns = ["CITATION.cff"]
|
|
57
|
+
scanner = "regex"
|
|
58
|
+
regex = '(?m)^abstract: >-\n(?P<content>(?:^ .*\n)+)'
|
|
59
|
+
snippets = ["abstract"]
|
|
60
|
+
render = "{{ content | plain | prefix(' ') }}\n"
|
|
61
|
+
|
|
62
|
+
# The keywords sequence of the citation metadata.
|
|
63
|
+
[[targets]]
|
|
64
|
+
patterns = ["CITATION.cff"]
|
|
65
|
+
snippets = ["keywords"]
|
|
66
|
+
render = "{{ content | prefix('- ') }}"
|
|
67
|
+
|
|
68
|
+
# The Zenodo metadata, which is JSON and therefore carries no markers.
|
|
69
|
+
[[targets]]
|
|
70
|
+
patterns = [".zenodo.json"]
|
|
71
|
+
scanner = "json"
|
|
72
|
+
insert = [
|
|
73
|
+
{ snippet = "abstract", pointer = "/description", render = "{{ content | plain | unwrap }}" },
|
|
74
|
+
{ snippet = "keywords", pointer = "/keywords", shape = "lines" },
|
|
75
|
+
]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## `tagline`
|
|
79
|
+
|
|
80
|
+
```text
|
|
81
|
+
Integrate SLURM jobs into a StepUp workflow
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## `abstract`
|
|
85
|
+
|
|
86
|
+
```markdown
|
|
87
|
+
StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
|
|
88
|
+
extension to integrate queued jobs into a workflow.
|
|
89
|
+
It currently supports integration with [SLURM](https://slurm.schedmd.com/)
|
|
90
|
+
and is designed to be extensible to other job schedulers.
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
## `keywords`
|
|
94
|
+
|
|
95
|
+
```text
|
|
96
|
+
asyncio
|
|
97
|
+
batch computing
|
|
98
|
+
build automation
|
|
99
|
+
build tool
|
|
100
|
+
extension
|
|
101
|
+
high-performance computing
|
|
102
|
+
HPC
|
|
103
|
+
job scheduler
|
|
104
|
+
job submission
|
|
105
|
+
parallel
|
|
106
|
+
POSIX
|
|
107
|
+
Python
|
|
108
|
+
queued jobs
|
|
109
|
+
reproducibility
|
|
110
|
+
reproducible research
|
|
111
|
+
research software
|
|
112
|
+
SLURM
|
|
113
|
+
StepUp
|
|
114
|
+
StepUp Core
|
|
115
|
+
supercomputer
|
|
116
|
+
workflow automation
|
|
117
|
+
workflow management
|
|
118
|
+
```
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
3
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
4
|
-->
|
|
5
|
+
|
|
5
6
|
# Job Files and the Submit-and-Wait Contract
|
|
6
7
|
|
|
7
8
|
## Job Lifecycle and Files
|
|
@@ -34,6 +35,17 @@ attaches to the job it already submitted, instead of submitting a second one:
|
|
|
34
35
|
This is why the digest is written into the log rather than derived at read time:
|
|
35
36
|
the log has to be interpretable by a process that never saw the original submission.
|
|
36
37
|
|
|
38
|
+
The invariant behind the contract is that at most one job runs per job directory,
|
|
39
|
+
because every job in it writes to the same `slurmjob.out`, `slurmjob.err` and `slurmjob.ret`.
|
|
40
|
+
Two places protect it:
|
|
41
|
+
|
|
42
|
+
- The `Submitting` marker is logged before `sbatch` runs and replaced by `Submitted <jobid>`
|
|
43
|
+
once the job ID is known.
|
|
44
|
+
A log that stops at the marker means the job ID may have been lost,
|
|
45
|
+
so the next run refuses to submit rather than risk a second job next to an untracked one.
|
|
46
|
+
- `onchange="resubmit"` waits for the cancelled job to leave the queue before resubmitting,
|
|
47
|
+
because `scancel` returns as soon as the request is accepted.
|
|
48
|
+
|
|
37
49
|
## sacct Caching
|
|
38
50
|
|
|
39
51
|
All concurrent `sq-sbatch-and-wait` processes share one cached `sacct` result per cluster,
|
stepup_queue-2.0.0rc3/stepup/queue/canceljobs.py → stepup_queue-2.0.1/stepup/queue/cancel_jobs.py
RENAMED
|
@@ -1,25 +1,24 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
2
2
|
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
3
|
-
"""
|
|
3
|
+
"""Cancel jobs that have been submitted to SLURM."""
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
6
|
import subprocess
|
|
7
7
|
import sys
|
|
8
|
-
from collections.abc import
|
|
8
|
+
from collections.abc import Sequence
|
|
9
9
|
|
|
10
10
|
from path import Path
|
|
11
11
|
from rich.console import Console
|
|
12
12
|
|
|
13
|
-
from stepup.core.config import ConfigLoader
|
|
14
|
-
|
|
15
13
|
from .log import read_jobid_cluster_status
|
|
16
14
|
from .utils import DONE_STATES, search_jobs
|
|
17
15
|
|
|
18
|
-
__all__ = ()
|
|
16
|
+
__all__ = ("cancel_jobs",)
|
|
19
17
|
|
|
20
18
|
|
|
21
|
-
def
|
|
19
|
+
def cancel_jobs(argv: Sequence[str] | None = None):
|
|
22
20
|
"""Iterate over all slurmjob.log files, read the SLURM job IDs, and cancel them."""
|
|
21
|
+
args = parse_args(argv)
|
|
23
22
|
console = Console(highlight=False)
|
|
24
23
|
if not args.commit:
|
|
25
24
|
console.print("[yellow]# Note: No jobs are actually cancelled.[/]")
|
|
@@ -41,18 +40,16 @@ def canceljobs_tool(args: argparse.Namespace):
|
|
|
41
40
|
# Cancel at most 100 at a time to avoid exceeding the command line length limit,
|
|
42
41
|
# and to play nice with SLURM.
|
|
43
42
|
while len(cluster_jobs) > 0:
|
|
44
|
-
|
|
43
|
+
batch = cluster_jobs[:100]
|
|
45
44
|
cluster_jobs[:] = cluster_jobs[100:]
|
|
46
45
|
|
|
47
46
|
command_args = ["scancel"]
|
|
48
47
|
if cluster is not None:
|
|
49
48
|
command_args.extend(["-M", cluster])
|
|
50
|
-
command_args.extend(str(job_id) for job_id, _, _ in
|
|
49
|
+
command_args.extend(str(job_id) for job_id, _, _ in batch)
|
|
51
50
|
|
|
52
51
|
# Using subprocess.run for better control and error handling
|
|
53
|
-
print_cancel_command(
|
|
54
|
-
console, [job_id for job_id, _, _ in cancel_jobs], cluster, None
|
|
55
|
-
)
|
|
52
|
+
print_cancel_command(console, [job_id for job_id, _, _ in batch], cluster, None)
|
|
56
53
|
result = subprocess.run(command_args, check=False)
|
|
57
54
|
all_good &= result.returncode == 0
|
|
58
55
|
else:
|
|
@@ -63,10 +60,11 @@ def canceljobs_tool(args: argparse.Namespace):
|
|
|
63
60
|
sys.exit(1)
|
|
64
61
|
|
|
65
62
|
|
|
66
|
-
def
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
63
|
+
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
|
64
|
+
"""Parse command-line arguments."""
|
|
65
|
+
parser = argparse.ArgumentParser(
|
|
66
|
+
prog="sq-cancel-jobs",
|
|
67
|
+
description="Cancel running jobs in the current StepUp workflow.",
|
|
70
68
|
)
|
|
71
69
|
parser.add_argument(
|
|
72
70
|
"paths",
|
|
@@ -90,13 +88,12 @@ def canceljobs_subcommand(subparsers, loader: ConfigLoader) -> Callable:
|
|
|
90
88
|
default=False,
|
|
91
89
|
help="Select all jobs, including the ones that seem to be done already.",
|
|
92
90
|
)
|
|
93
|
-
|
|
94
|
-
return canceljobs_tool
|
|
91
|
+
return parser.parse_args(argv)
|
|
95
92
|
|
|
96
93
|
|
|
97
94
|
def print_cancel_command(
|
|
98
95
|
console: Console, job_ids: list[int], cluster: str | None, comment: str | None
|
|
99
|
-
)
|
|
96
|
+
):
|
|
100
97
|
"""Print the job cancellation command."""
|
|
101
98
|
parts = ["[green]scancel[/]"]
|
|
102
99
|
if cluster is not None:
|
|
@@ -105,3 +102,7 @@ def print_cancel_command(
|
|
|
105
102
|
if comment is not None:
|
|
106
103
|
parts.append(f" [bright_black]# {comment}[/]")
|
|
107
104
|
console.print(" ".join(parts))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
if __name__ == "__main__":
|
|
108
|
+
sys.exit(cancel_jobs())
|