stepup-queue 2.0.0rc4__tar.gz → 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/.zenodo.json +15 -11
  2. stepup_queue-2.1.0/CITATION.cff +68 -0
  3. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/CLAUDE.md +21 -4
  4. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/MANIFEST.in +1 -1
  5. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/PKG-INFO +19 -17
  6. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/README.md +5 -3
  7. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/REUSE.toml +5 -6
  8. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/pyproject.toml +57 -21
  9. stepup_queue-2.1.0/snipwise.md +118 -0
  10. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/CLAUDE.md +14 -0
  11. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/__init__.py +1 -1
  12. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/api.py +21 -1
  13. stepup_queue-2.0.0rc4/stepup/queue/canceljobs.py → stepup_queue-2.1.0/stepup/queue/cancel_jobs.py +20 -18
  14. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/log.py +66 -12
  15. stepup_queue-2.0.0rc4/stepup/queue/removejobs.py → stepup_queue-2.1.0/stepup/queue/remove_jobs.py +33 -14
  16. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/sbatch.py +223 -54
  17. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup/queue/utils.py +5 -3
  18. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/PKG-INFO +19 -17
  19. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/SOURCES.txt +4 -2
  20. stepup_queue-2.1.0/stepup_queue.egg-info/entry_points.txt +4 -0
  21. stepup_queue-2.1.0/stepup_queue.egg-info/requires.txt +15 -0
  22. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/scm_file_list.json +15 -7
  23. stepup_queue-2.1.0/stepup_queue.egg-info/scm_version.json +8 -0
  24. stepup_queue-2.0.0rc4/stepup_queue.egg-info/entry_points.txt +0 -6
  25. stepup_queue-2.0.0rc4/stepup_queue.egg-info/requires.txt +0 -16
  26. stepup_queue-2.0.0rc4/stepup_queue.egg-info/scm_version.json +0 -8
  27. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSE +0 -0
  28. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSES/CC-BY-SA-4.0.txt +0 -0
  29. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSES/LGPL-3.0-or-later.txt +0 -0
  30. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
  31. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/setup.cfg +0 -0
  32. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/dependency_links.txt +0 -0
  33. {stepup_queue-2.0.0rc4 → stepup_queue-2.1.0}/stepup_queue.egg-info/top_level.txt +0 -0
@@ -18,24 +18,28 @@
18
18
  ],
19
19
  "language": "eng",
20
20
  "keywords": [
21
- "SLURM",
21
+ "asyncio",
22
+ "batch computing",
23
+ "build automation",
24
+ "build tool",
25
+ "extension",
26
+ "high-performance computing",
27
+ "HPC",
22
28
  "job scheduler",
23
29
  "job submission",
30
+ "parallel",
31
+ "POSIX",
32
+ "Python",
24
33
  "queued jobs",
25
- "batch computing",
26
- "high-performance computing",
27
- "supercomputer",
28
- "workflow automation",
29
- "workflow management",
30
- "build tool",
31
- "build automation",
32
34
  "reproducibility",
33
35
  "reproducible research",
34
36
  "research software",
35
- "extension",
37
+ "SLURM",
38
+ "StepUp",
36
39
  "StepUp Core",
37
- "POSIX",
38
- "Python"
40
+ "supercomputer",
41
+ "workflow automation",
42
+ "workflow management"
39
43
  ],
40
44
  "related_identifiers": [
41
45
  {
@@ -0,0 +1,68 @@
1
+ # SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
2
+ # SPDX-License-Identifier: LGPL-3.0-or-later
3
+ cff-version: 1.2.0
4
+ message: If you use this software, please cite it using these metadata.
5
+ title: StepUp Queue
6
+ type: software
7
+ url: https://reproducible-reporting.github.io/stepup-queue
8
+ abstract: >-
9
+ StepUp Queue is an experimental StepUp
10
+ extension to integrate queued jobs into a workflow.
11
+ It currently supports integration with SLURM
12
+ and is designed to be extensible to other job schedulers.
13
+ authors:
14
+ - family-names: Verstraelen
15
+ given-names: Toon
16
+ orcid: https://orcid.org/0000-0001-9288-5608
17
+ affiliation: Center for Molecular Modeling (CMM), Ghent University
18
+ address: Technologiepark-Zwijnaarde 46
19
+ post-code: 9052
20
+ city: Zwijnaarde
21
+ country: BE
22
+ license: LGPL-3.0-or-later
23
+ repository-code: https://github.com/reproducible-reporting/stepup-queue
24
+ keywords:
25
+ # snipwise.md BEGIN keywords
26
+ - asyncio
27
+ - batch computing
28
+ - build automation
29
+ - build tool
30
+ - extension
31
+ - high-performance computing
32
+ - HPC
33
+ - job scheduler
34
+ - job submission
35
+ - parallel
36
+ - POSIX
37
+ - Python
38
+ - queued jobs
39
+ - reproducibility
40
+ - reproducible research
41
+ - research software
42
+ - SLURM
43
+ - StepUp
44
+ - StepUp Core
45
+ - supercomputer
46
+ - workflow automation
47
+ - workflow management
48
+ # snipwise.md END keywords
49
+ identifiers:
50
+ - type: doi
51
+ value: 10.5281/zenodo.22024209
52
+ description: Concept DOI, which always resolves to the latest archived version.
53
+ references:
54
+ - title: StepUp Core
55
+ type: software
56
+ authors:
57
+ - family-names: Verstraelen
58
+ given-names: Toon
59
+ orcid: https://orcid.org/0000-0001-9288-5608
60
+ affiliation: Center for Molecular Modeling (CMM), Ghent University
61
+ url: https://reproducible-reporting.github.io/stepup-core
62
+ repository-code: https://github.com/reproducible-reporting/stepup-core
63
+ license: LGPL-3.0-or-later
64
+ identifiers:
65
+ - type: doi
66
+ value: 10.5281/zenodo.22018869
67
+ description: Concept DOI, which always resolves to the latest archived version.
68
+ notes: StepUp Queue is an extension of StepUp Core and requires it at runtime.
@@ -12,6 +12,7 @@ Guidance that applies to only part of the repo lives next to the code it governs
12
12
 
13
13
  - `stepup/queue/CLAUDE.md`: job file conventions, the submit-and-wait contract, sacct caching.
14
14
  - `tests/CLAUDE.md`: test layout and conventions.
15
+ - `docs/CLAUDE.md`: regenerating the social card.
15
16
 
16
17
  ## Project Overview
17
18
 
@@ -31,10 +32,11 @@ The package is organized around two layers:
31
32
  Its `sbatch()` function calls `stepup.core.api.run()`
32
33
  to register a `sq-sbatch-and-wait` step with StepUp Core.
33
34
 
34
- - **Individual command modules** each implement a `main()` function that serves as a CLI tool.
35
- These are registered via `[project.scripts]` and the `stepup.tools` entry points
36
- in `pyproject.toml`, and are invoked by StepUp steps as external commands.
37
- When StepUp executes such a step, the command runs in the working directory of the job.
35
+ - **Individual command modules** each implement a function that serves as a CLI entry point.
36
+ These are registered via `[project.scripts]` in `pyproject.toml`.
37
+ `sq-sbatch-and-wait` is invoked by StepUp steps as an external command,
38
+ and runs in the working directory of the job.
39
+ `sq-cancel-jobs` and `sq-remove-jobs` are run by the user from the shell.
38
40
 
39
41
  ## Coding Conventions
40
42
 
@@ -43,6 +45,21 @@ The package is organized around two layers:
43
45
  Use "which"/"because"/"that", or split the sentence.
44
46
  - **Do not add `# noqa`**
45
47
  unless the violation is a genuine false positive that cannot be resolved by restructuring.
48
+ - **A path is a string with `/` as its only separator, also on Windows.**
49
+ Import `Path` from `stepup.core.path`, not from `path`, and avoid `os.path`.
50
+ `test_slash_paths` of `ConventionTests` enforces this.
51
+ StepUp Queue only supports POSIX systems,
52
+ but it follows the path and text file conventions of StepUp Core all the same.
53
+
54
+ ### Text Files
55
+
56
+ Open every text file with `encoding="utf-8"`, and write text files with `newline="\n"` as well,
57
+ so that files are identical on all platforms,
58
+ including Windows with Python 3.14 and older, where the default is the ANSI code page.
59
+ Ruff enforces the encoding with `PLW1514`,
60
+ but only for `open()` and `pathlib`, not for the methods of `Path`.
61
+ In the package, `read_text()` and `write_text()` of `Path` need `encoding="utf-8"` too.
62
+ The same holds for `subprocess.run()` with `text=True`, which ruff does not check either.
46
63
 
47
64
  ### Docstrings
48
65
 
@@ -9,7 +9,7 @@ prune overrides
9
9
  exclude .editorconfig
10
10
  exclude .gitignore
11
11
  exclude .pre-commit-config.yaml
12
- exclude mkdocs.yaml
12
+ exclude zensical.toml
13
13
 
14
14
  # Exclude tests for now. (Could be useful later for conda package.)
15
15
  prune tests
@@ -1,13 +1,14 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup-queue
3
- Version: 2.0.0rc4
4
- Summary: StepUp Queue integrates queued jobs into a StepUp workflow.
3
+ Version: 2.1.0
4
+ Summary: Integrate SLURM jobs into a StepUp workflow
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
7
7
  Project-URL: Documentation, https://reproducible-reporting.github.io/stepup-queue/
8
8
  Project-URL: Issues, https://github.com/reproducible-reporting/stepup-queue/issues
9
9
  Project-URL: Source, https://github.com/reproducible-reporting/stepup-queue/
10
10
  Project-URL: Changelog, https://reproducible-reporting.github.io/stepup-queue/changelog/
11
+ Keywords: asyncio,batch computing,build automation,build tool,extension,high-performance computing,HPC,job scheduler,job submission,parallel,POSIX,Python,queued jobs,reproducibility,reproducible research,research software,SLURM,StepUp,StepUp Core,supercomputer,workflow automation,workflow management
11
12
  Classifier: Development Status :: 4 - Beta
12
13
  Classifier: Environment :: Console
13
14
  Classifier: Intended Audience :: Education
@@ -20,25 +21,24 @@ Classifier: Programming Language :: Python :: 3.11
20
21
  Classifier: Programming Language :: Python :: 3.12
21
22
  Classifier: Programming Language :: Python :: 3.13
22
23
  Classifier: Programming Language :: Python :: 3.14
24
+ Classifier: Programming Language :: Python :: 3.15
23
25
  Classifier: Topic :: Software Development :: Build Tools
24
26
  Requires-Python: >=3.11
25
27
  Description-Content-Type: text/markdown
26
28
  License-File: LICENSE
27
- Requires-Dist: path>=16.14.0
28
29
  Requires-Dist: rich>=13.0.0
29
- Requires-Dist: stepup<5.0.0a1,>=4.0.0rc12
30
+ Requires-Dist: stepup<5.0.0a1,>=4.1.0
31
+ Provides-Extra: tests
32
+ Requires-Dist: pytest>=8.2.0; extra == "tests"
33
+ Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
34
+ Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
35
+ Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
30
36
  Provides-Extra: dev
31
- Requires-Dist: cffconvert; extra == "dev"
32
- Requires-Dist: mike; extra == "dev"
33
- Requires-Dist: mkdocs; extra == "dev"
34
- Requires-Dist: mkdocstrings[python]; extra == "dev"
35
- Requires-Dist: mkdocs-macros-plugin; extra == "dev"
36
- Requires-Dist: mkdocs-material; extra == "dev"
37
- Requires-Dist: psutil; extra == "dev"
38
- Requires-Dist: pytest; extra == "dev"
39
- Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
40
- Requires-Dist: pytest-xdist; extra == "dev"
37
+ Requires-Dist: stepup-queue[tests]; extra == "dev"
38
+ Requires-Dist: mkdocstrings-python; extra == "dev"
41
39
  Requires-Dist: reuse; extra == "dev"
40
+ Requires-Dist: snipwise<2.0a0; extra == "dev"
41
+ Requires-Dist: zensical; extra == "dev"
42
42
  Dynamic: license-file
43
43
 
44
44
  <!--
@@ -50,15 +50,17 @@ SPDX-License-Identifier: LGPL-3.0-or-later
50
50
 
51
51
  [![release](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml)
52
52
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
53
- [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
53
+ [![docs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml)
54
54
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
55
55
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
56
56
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
57
57
  ![LGPL-3 License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)
58
58
 
59
+ <!-- snipwise.md BEGIN abstract -->
59
60
  StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
60
61
  extension to integrate queued jobs into a workflow.
61
- Currently, it only supports integration with [SLURM](https://slurm.schedmd.com/),
62
- but it is designed to be extensible to other job schedulers.
62
+ It currently supports integration with [SLURM](https://slurm.schedmd.com/)
63
+ and is designed to be extensible to other job schedulers.
64
+ <!-- snipwise.md END abstract -->
63
65
 
64
66
  For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).
@@ -7,15 +7,17 @@ SPDX-License-Identifier: LGPL-3.0-or-later
7
7
 
8
8
  [![release](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/release.yaml)
9
9
  [![pytest](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/pytest.yaml)
10
- [![mkdocs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/mkdocs.yaml)
10
+ [![docs](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-queue/actions/workflows/docs.yaml)
11
11
  [![PyPI Version](https://img.shields.io/pypi/v/stepup-queue)](https://pypi.org/project/stepup-queue/)
12
12
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22024209.svg)](https://doi.org/10.5281/zenodo.22024209)
13
13
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup-queue)
14
14
  ![LGPL-3 License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)
15
15
 
16
+ <!-- snipwise.md BEGIN abstract -->
16
17
  StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
17
18
  extension to integrate queued jobs into a workflow.
18
- Currently, it only supports integration with [SLURM](https://slurm.schedmd.com/),
19
- but it is designed to be extensible to other job schedulers.
19
+ It currently supports integration with [SLURM](https://slurm.schedmd.com/)
20
+ and is designed to be extensible to other job schedulers.
21
+ <!-- snipwise.md END abstract -->
20
22
 
21
23
  For more information, consult the [documentation](https://reproducible-reporting.github.io/stepup-queue).
@@ -15,11 +15,10 @@ SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
15
15
  SPDX-License-Identifier = "LGPL-3.0-or-later"
16
16
 
17
17
  [[annotations]]
18
- path = "docs/clean_stdout.sed"
19
- SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
20
- SPDX-License-Identifier = "LGPL-3.0-or-later"
21
-
22
- [[annotations]]
23
- path = "docs/logo.svg"
18
+ path = [
19
+ "docs/logo.svg",
20
+ "docs/social-card.jpg",
21
+ "docs/social-card.svg",
22
+ ]
24
23
  SPDX-FileCopyrightText = "2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
25
24
  SPDX-License-Identifier = "LicenseRef-StepUp-Logo-v1"
@@ -1,7 +1,7 @@
1
1
  # SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
3
  [build-system]
4
- requires = ["setuptools>=65.0", "setuptools_scm>=8.1"]
4
+ requires = ["setuptools>=77.0", "setuptools_scm>=8.1"]
5
5
  build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
@@ -9,10 +9,36 @@ name = "stepup-queue"
9
9
  authors = [
10
10
  { name="Toon Verstraelen", email="toon.verstraelen@ugent.be" },
11
11
  ]
12
- description = "StepUp Queue integrates queued jobs into a StepUp workflow."
12
+ description = "Integrate SLURM jobs into a StepUp workflow"
13
13
  readme = "README.md"
14
14
  license = "LGPL-3.0-or-later"
15
15
  requires-python = ">=3.11"
16
+ keywords = [
17
+ # snipwise.md BEGIN keywords
18
+ "asyncio",
19
+ "batch computing",
20
+ "build automation",
21
+ "build tool",
22
+ "extension",
23
+ "high-performance computing",
24
+ "HPC",
25
+ "job scheduler",
26
+ "job submission",
27
+ "parallel",
28
+ "POSIX",
29
+ "Python",
30
+ "queued jobs",
31
+ "reproducibility",
32
+ "reproducible research",
33
+ "research software",
34
+ "SLURM",
35
+ "StepUp",
36
+ "StepUp Core",
37
+ "supercomputer",
38
+ "workflow automation",
39
+ "workflow management",
40
+ # snipwise.md END keywords
41
+ ]
16
42
  classifiers = [
17
43
  "Development Status :: 4 - Beta",
18
44
  "Environment :: Console",
@@ -26,29 +52,32 @@ classifiers = [
26
52
  "Programming Language :: Python :: 3.12",
27
53
  "Programming Language :: Python :: 3.13",
28
54
  "Programming Language :: Python :: 3.14",
55
+ "Programming Language :: Python :: 3.15",
29
56
  "Topic :: Software Development :: Build Tools",
30
57
  ]
31
58
  dependencies = [
32
- # Ensure changes to these dependencies are reflected in .github/requirements-old.txt
33
- "path>=16.14.0",
34
59
  "rich>=13.0.0",
35
- "stepup>=4.0.0rc12,<5.0.0a1",
60
+ "stepup>=4.1.0,<5.0.0a1",
36
61
  ]
37
62
  dynamic = ["version"]
38
63
 
39
64
  [project.optional-dependencies]
40
- dev = [
41
- "cffconvert",
42
- "mike",
43
- "mkdocs",
44
- "mkdocstrings[python]",
45
- "mkdocs-macros-plugin",
46
- "mkdocs-material",
47
- "psutil",
48
- "pytest",
65
+ # What the test suite needs, and nothing more.
66
+ # The pytest workflow installs this extra rather than `dev`,
67
+ # because it resolves the oldest supported versions of the direct dependencies,
68
+ # which is only meaningful for packages that the tests actually exercise.
69
+ tests = [
70
+ "pytest>=8.2.0",
49
71
  "pytest-asyncio>=0.24",
50
- "pytest-xdist",
72
+ "pytest-timeout>=2.3.1",
73
+ "pytest-xdist>=3.6.1",
74
+ ]
75
+ dev = [
76
+ "stepup-queue[tests]",
77
+ "mkdocstrings-python",
51
78
  "reuse",
79
+ "snipwise<2.0a0",
80
+ "zensical",
52
81
  ]
53
82
 
54
83
  [project.urls]
@@ -58,24 +87,28 @@ Source = "https://github.com/reproducible-reporting/stepup-queue/"
58
87
  Changelog = "https://reproducible-reporting.github.io/stepup-queue/changelog/"
59
88
 
60
89
  [project.scripts]
90
+ sq-cancel-jobs = "stepup.queue.cancel_jobs:cancel_jobs"
91
+ sq-remove-jobs = "stepup.queue.remove_jobs:remove_jobs"
61
92
  sq-sbatch-and-wait = "stepup.queue.sbatch:sbatch"
62
93
 
63
- [project.entry-points."stepup.tools"]
64
- canceljobs = "stepup.queue.canceljobs:canceljobs_subcommand"
65
- removejobs = "stepup.queue.removejobs:removejobs_subcommand"
66
-
67
94
  [tool.pytest.ini_options]
68
95
  addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
69
96
  asyncio_default_fixture_loop_scope = "function"
97
+ timeout = 15
70
98
 
71
99
  [tool.ruff]
72
100
  line-length = 100
73
101
  target-version = "py311"
74
102
 
75
103
  [tool.ruff.lint]
104
+ # Preview mode is only enabled for the preview rules selected by their full code below,
105
+ # but it also turns on the preview behavior of stable rules.
106
+ preview = true
107
+ explicit-preview-rules = true
76
108
  select = [
77
109
  "A", "B", "BLE", "C4", "E", "EXE", "F", "I", "ICN", "ISC", "N", "NPY", "PERF", "PIE", "PL",
78
- "PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W"
110
+ "PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W",
111
+ "PLW1514", # https://docs.astral.sh/ruff/rules/unspecified-encoding/ (preview in ruff 0.17.2)
79
112
  ]
80
113
  ignore = [
81
114
  "PLR0904", # https://docs.astral.sh/ruff/rules/too-many-public-methods/
@@ -109,7 +142,10 @@ local_scheme = "no-local-version"
109
142
  [tool.pymarkdown]
110
143
  # Report unknown or mistyped settings instead of silently ignoring them.
111
144
  mode.strict-config = true
112
- # The documentation indents nested lists by four spaces, matching Python-Markdown and MkDocs.
145
+ # The documentation pages carry YAML front matter,
146
+ # which pymarkdown parses as a setext heading unless this extension is on.
147
+ extensions.front-matter.enabled = true
148
+ # The documentation indents nested lists by four spaces, matching Python-Markdown and Zensical.
113
149
  plugins.md007.indent = 4
114
150
  plugins.md013.line_length = 100
115
151
  plugins.md013.heading_line_length = 100
@@ -0,0 +1,118 @@
1
+ <!--
2
+ SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
+ SPDX-License-Identifier: LGPL-3.0-or-later
4
+ -->
5
+
6
+ # Snipwise Configuration
7
+
8
+ Consult the full documentation at <https://reproducible-reporting.github.io/snipwise/>.
9
+
10
+ ```toml
11
+ # The Markdown files show the abstract as it is written below, links and all.
12
+ [[targets]]
13
+ patterns = ["README.md", "docs/index.md"]
14
+
15
+ # The summary of the Python package metadata, which is a single-line TOML string.
16
+ [[targets]]
17
+ patterns = ["pyproject.toml"]
18
+ scanner = "regex"
19
+ regex = '(?m)^description = "(?P<content>[^"]*)"$'
20
+ snippets = ["tagline"]
21
+ render = "{{ content | unwrap }}"
22
+
23
+ # The meta description that every documentation page carries.
24
+ [[targets]]
25
+ patterns = ["zensical.toml"]
26
+ scanner = "regex"
27
+ regex = '(?m)^site_description = "(?P<content>[^"]*)"$'
28
+ snippets = ["tagline"]
29
+ render = "{{ content | unwrap }}"
30
+
31
+ # The docstring of the package, which is a single-line string.
32
+ [[targets]]
33
+ patterns = ["stepup/queue/__init__.py"]
34
+ scanner = "regex"
35
+ regex = '(?m)^"""(?P<content>[^"]*)"""$'
36
+ snippets = ["tagline"]
37
+ render = "{{ content | unwrap | suffix('.') }}"
38
+
39
+ # The tagline in the social card, whose tspan carries the id that locates it.
40
+ [[targets]]
41
+ patterns = ["docs/social-card.svg"]
42
+ scanner = "regex"
43
+ regex = '<tspan\b[^>]*\bid="tagline"[^>]*>(?P<content>[^<]*)</tspan>'
44
+ snippets = ["tagline"]
45
+ render = "{{ content | unwrap }}"
46
+
47
+ # The keywords array of the Python package metadata.
48
+ [[targets]]
49
+ patterns = ["pyproject.toml"]
50
+ snippets = ["keywords"]
51
+ render = '''{{ content | prefix('"') | suffix('",') }}'''
52
+
53
+ # The abstract of the citation metadata, which is a folded YAML block scalar.
54
+ # The template terminates the last line, because the region includes its newline.
55
+ [[targets]]
56
+ patterns = ["CITATION.cff"]
57
+ scanner = "regex"
58
+ regex = '(?m)^abstract: >-\n(?P<content>(?:^ .*\n)+)'
59
+ snippets = ["abstract"]
60
+ render = "{{ content | plain | prefix(' ') }}\n"
61
+
62
+ # The keywords sequence of the citation metadata.
63
+ [[targets]]
64
+ patterns = ["CITATION.cff"]
65
+ snippets = ["keywords"]
66
+ render = "{{ content | prefix('- ') }}"
67
+
68
+ # The Zenodo metadata, which is JSON and therefore carries no markers.
69
+ [[targets]]
70
+ patterns = [".zenodo.json"]
71
+ scanner = "json"
72
+ insert = [
73
+ { snippet = "abstract", pointer = "/description", render = "{{ content | plain | unwrap }}" },
74
+ { snippet = "keywords", pointer = "/keywords", shape = "lines" },
75
+ ]
76
+ ```
77
+
78
+ ## `tagline`
79
+
80
+ ```text
81
+ Integrate SLURM jobs into a StepUp workflow
82
+ ```
83
+
84
+ ## `abstract`
85
+
86
+ ```markdown
87
+ StepUp Queue is an experimental [StepUp](https://reproducible-reporting.github.io/stepup-core)
88
+ extension to integrate queued jobs into a workflow.
89
+ It currently supports integration with [SLURM](https://slurm.schedmd.com/)
90
+ and is designed to be extensible to other job schedulers.
91
+ ```
92
+
93
+ ## `keywords`
94
+
95
+ ```text
96
+ asyncio
97
+ batch computing
98
+ build automation
99
+ build tool
100
+ extension
101
+ high-performance computing
102
+ HPC
103
+ job scheduler
104
+ job submission
105
+ parallel
106
+ POSIX
107
+ Python
108
+ queued jobs
109
+ reproducibility
110
+ reproducible research
111
+ research software
112
+ SLURM
113
+ StepUp
114
+ StepUp Core
115
+ supercomputer
116
+ workflow automation
117
+ workflow management
118
+ ```
@@ -31,10 +31,24 @@ attaches to the job it already submitted, instead of submitting a second one:
31
31
  3. With a log whose digest matches, it resumes waiting for the existing job.
32
32
  4. With a digest mismatch, the `onchange` policy decides
33
33
  between `raise`, `resubmit` and `ignore`.
34
+ With `raise` or `resubmit`, an optional `complete` command is consulted first.
35
+ When it succeeds, the new digest is written into the log and the existing job is kept,
36
+ which brings the step back to case 3.
34
37
 
35
38
  This is why the digest is written into the log rather than derived at read time:
36
39
  the log has to be interpretable by a process that never saw the original submission.
37
40
 
41
+ The invariant behind the contract is that at most one job runs per job directory,
42
+ because every job in it writes to the same `slurmjob.out`, `slurmjob.err` and `slurmjob.ret`.
43
+ Two places protect it:
44
+
45
+ - The `Submitting` marker is logged before `sbatch` runs and replaced by `Submitted <jobid>`
46
+ once the job ID is known.
47
+ A log that stops at the marker means the job ID may have been lost,
48
+ so the next run refuses to submit rather than risk a second job next to an untracked one.
49
+ - `onchange="resubmit"` waits for the cancelled job to leave the queue before resubmitting,
50
+ because `scancel` returns as soon as the request is accepted.
51
+
38
52
  ## sacct Caching
39
53
 
40
54
  All concurrent `sq-sbatch-and-wait` processes share one cached `sacct` result per cluster,
@@ -1,5 +1,5 @@
1
1
  # SPDX-FileCopyrightText: 2025 Toon Verstraelen <Toon.Verstraelen@UGent.be>
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
- """StepUp Queue package."""
3
+ """Integrate SLURM jobs into a StepUp workflow."""
4
4
 
5
5
  __all__ = ()
@@ -21,6 +21,7 @@ def sbatch(
21
21
  out: Collection[StrPath] | StrPath = (),
22
22
  vol: Collection[StrPath] | StrPath = (),
23
23
  onchange: str | None = None,
24
+ complete: str | None = None,
24
25
  optional: bool = False,
25
26
  resources: dict[str, int] | str | None = None,
26
27
  duration: float | None = None,
@@ -63,8 +64,23 @@ def sbatch(
63
64
  and pass it here as `source rc.sh`.
64
65
  In this case, you usually also want to include `rc.sh` in the `inp` list.
65
66
  onchange
66
- Policy when a the inputs of a previously submitted job have changed.
67
+ Policy when the inputs of a previously submitted job have changed.
67
68
  Must be one of `"raise"`, `"resubmit"` or `"ignore"`.
69
+ When not given, the environment variable `STEPUP_QUEUE_ONCHANGE` is used,
70
+ with `"raise"` as default.
71
+ complete
72
+ A shell command that tells whether the outputs of the job are complete,
73
+ e.g. `"test -f result.txt"`.
74
+ The command is executed in the working directory
75
+ when the inputs of a previously submitted job have changed.
76
+ A zero exit code keeps the existing job, as if its inputs had not changed.
77
+ A non-zero exit code applies the `onchange` policy.
78
+ Hence, with `onchange="resubmit"`, a job whose inputs have changed is only resubmitted
79
+ when outputs are missing.
80
+ This is useful for job scripts that skip work whose outputs already exist.
81
+ With `onchange="raise"`, an error is only raised when outputs are missing.
82
+ The command is not used with `onchange="ignore"`,
83
+ because that policy keeps the existing job anyway.
68
84
  """
69
85
  if ext == "":
70
86
  ext = ".sh"
@@ -81,6 +97,10 @@ def sbatch(
81
97
  if onchange not in ["raise", "resubmit", "ignore"]:
82
98
  raise ValueError(f"Invalid onchange policy {onchange}.")
83
99
  cmd += f" --onchange={onchange}"
100
+ if complete is not None:
101
+ if complete.strip() == "":
102
+ raise ValueError("The complete command must not be empty.")
103
+ cmd += f" --complete={shlex.quote(complete)}"
84
104
  return run(
85
105
  cmd,
86
106
  inp=[f"slurmjob{ext}", *coerce_paths(inp)],