stepup 4.0.0rc13__tar.gz → 4.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. stepup-4.0.1/.zenodo.json +64 -0
  2. {stepup-4.0.0rc13 → stepup-4.0.1}/CITATION.cff +13 -8
  3. {stepup-4.0.0rc13 → stepup-4.0.1}/CLAUDE.md +18 -11
  4. stepup-4.0.1/CONTRIBUTING.md +44 -0
  5. {stepup-4.0.0rc13 → stepup-4.0.1}/PKG-INFO +18 -15
  6. {stepup-4.0.0rc13 → stepup-4.0.1}/README.md +4 -3
  7. {stepup-4.0.0rc13 → stepup-4.0.1}/REUSE.toml +7 -0
  8. {stepup-4.0.0rc13 → stepup-4.0.1}/pyproject.toml +61 -14
  9. stepup-4.0.1/snipwise.md +122 -0
  10. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/CLAUDE.md +14 -5
  11. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/__init__.py +1 -1
  12. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/__main__.py +4 -2
  13. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/builder.py +72 -17
  14. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/director.py +8 -7
  15. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/enums.py +37 -11
  16. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/executor.py +20 -28
  17. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/finalize.py +31 -15
  18. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/hash.py +51 -42
  19. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/hash_queue.py +59 -28
  20. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/render_jinja.py +7 -5
  21. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/startup.py +23 -31
  22. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/step.py +0 -9
  23. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/watcher.py +14 -12
  24. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/workflow.py +395 -210
  25. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/PKG-INFO +18 -15
  26. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/SOURCES.txt +5 -2
  27. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/requires.txt +13 -9
  28. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/scm_file_list.json +9 -5
  29. stepup-4.0.1/stepup.egg-info/scm_version.json +8 -0
  30. stepup-4.0.1/tools/make_stable_redirects.py +119 -0
  31. stepup-4.0.0rc13/.claude/skills/release/SKILL.md +0 -14
  32. stepup-4.0.0rc13/stepup.egg-info/scm_version.json +0 -8
  33. {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSE +0 -0
  34. {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSES/CC-BY-SA-4.0.txt +0 -0
  35. {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSES/LGPL-3.0-or-later.txt +0 -0
  36. {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
  37. {stepup-4.0.0rc13 → stepup-4.0.1}/MANIFEST.in +0 -0
  38. {stepup-4.0.0rc13 → stepup-4.0.1}/setup.cfg +0 -0
  39. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/api.py +0 -0
  40. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/asyncio.py +0 -0
  41. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/browse.py +0 -0
  42. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/call.py +0 -0
  43. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/cattrs.py +0 -0
  44. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/cgroups.py +0 -0
  45. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/clean.py +0 -0
  46. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/config_loader.py +0 -0
  47. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/config_tool.py +0 -0
  48. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/constants.py +0 -0
  49. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/exceptions.py +0 -0
  50. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/extapi.py +0 -0
  51. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/file.py +0 -0
  52. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/interact.py +0 -0
  53. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/job.py +0 -0
  54. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/logo.svg +0 -0
  55. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/nglob.py +0 -0
  56. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/outcome.py +0 -0
  57. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/path.py +0 -0
  58. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/pending.py +0 -0
  59. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/pytest.py +0 -0
  60. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/reporter.py +0 -0
  61. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/rpc.py +0 -0
  62. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/run.py +0 -0
  63. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/scheduler.py +0 -0
  64. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/script.py +0 -0
  65. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/sqlite3.py +0 -0
  66. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/static_tree.py +0 -0
  67. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/status.py +0 -0
  68. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/stepinfo.py +0 -0
  69. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/tool.py +0 -0
  70. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/tracebacks.py +0 -0
  71. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/trellis.py +0 -0
  72. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/tui.py +0 -0
  73. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/usage.py +0 -0
  74. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/utils.py +0 -0
  75. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/dependency_links.txt +0 -0
  76. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/entry_points.txt +0 -0
  77. {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/top_level.txt +0 -0
  78. {stepup-4.0.0rc13 → stepup-4.0.1}/tools/.gitignore +0 -0
  79. {stepup-4.0.0rc13 → stepup-4.0.1}/tools/analyze_perf.py +0 -0
  80. {stepup-4.0.0rc13 → stepup-4.0.1}/tools/leaves.py +0 -0
@@ -0,0 +1,64 @@
1
+ {
2
+ "upload_type": "software",
3
+ "title": "StepUp Core",
4
+ "description": "StepUp is a dynamic build tool and a modern alternative to Make. Its defining feature is that workflow generation and execution are unified: a plan.py script defines the initial build steps. While the workflow is being executed, any step can add more steps and dependencies, based on the outputs built so far. This makes StepUp ideal for builds where the full set of dependencies cannot be determined in advance.",
5
+ "creators": [
6
+ {
7
+ "name": "Verstraelen, Toon",
8
+ "orcid": "0000-0001-9288-5608",
9
+ "affiliation": "Center for Molecular Modeling (CMM), Ghent University"
10
+ }
11
+ ],
12
+ "license": "lgpl-3.0-or-later",
13
+ "access_right": "open",
14
+ "communities": [
15
+ {
16
+ "identifier": "ugent"
17
+ }
18
+ ],
19
+ "language": "eng",
20
+ "keywords": [
21
+ "asyncio",
22
+ "build automation",
23
+ "build tool",
24
+ "dynamic dependencies",
25
+ "incremental build",
26
+ "parallel",
27
+ "POSIX",
28
+ "Python",
29
+ "reproducibility",
30
+ "reproducible research",
31
+ "StepUp",
32
+ "terminal application",
33
+ "workflow",
34
+ "workflow management"
35
+ ],
36
+ "related_identifiers": [
37
+ {
38
+ "identifier": "https://reproducible-reporting.github.io/stepup-core",
39
+ "relation": "isDocumentedBy",
40
+ "resource_type": "publication-softwaredocumentation"
41
+ },
42
+ {
43
+ "identifier": "10.5281/zenodo.22024042",
44
+ "relation": "isRequiredBy",
45
+ "resource_type": "software"
46
+ },
47
+ {
48
+ "identifier": "10.5281/zenodo.22024209",
49
+ "relation": "isRequiredBy",
50
+ "resource_type": "software"
51
+ }
52
+ ],
53
+ "custom": {
54
+ "code:codeRepository": "https://github.com/reproducible-reporting/stepup-core/",
55
+ "code:programmingLanguage": [
56
+ {
57
+ "id": "python"
58
+ }
59
+ ],
60
+ "code:developmentStatus": {
61
+ "id": "active"
62
+ }
63
+ }
64
+ }
@@ -6,7 +6,8 @@ title: StepUp Core
6
6
  type: software
7
7
  url: https://reproducible-reporting.github.io/stepup-core
8
8
  abstract: >-
9
- StepUp is a dynamic build tool and a modern alternative to Make.
9
+ StepUp is a dynamic build tool and a modern alternative to
10
+ Make.
10
11
  Its defining feature is that workflow generation and execution are unified:
11
12
  a plan.py script defines the initial build steps.
12
13
  While the workflow is being executed, any step can add more steps and dependencies,
@@ -22,21 +23,25 @@ authors:
22
23
  post-code: 9052
23
24
  city: Zwijnaarde
24
25
  country: BE
26
+ license: LGPL-3.0-or-later
27
+ repository-code: https://github.com/reproducible-reporting/stepup-core
25
28
  keywords:
29
+ # snipwise.md BEGIN keywords
26
30
  - asyncio
27
- - build tool
28
31
  - build automation
29
- - workflow management
32
+ - build tool
30
33
  - dynamic dependencies
31
34
  - incremental build
32
- - reproducible research
35
+ - parallel
36
+ - POSIX
37
+ - Python
33
38
  - reproducibility
39
+ - reproducible research
40
+ - StepUp
34
41
  - terminal application
35
42
  - workflow
36
- - POSIX
37
- - Python
38
- license: LGPL-3.0-or-later
39
- repository-code: https://github.com/reproducible-reporting/stepup-core
43
+ - workflow management
44
+ # snipwise.md END keywords
40
45
  identifiers:
41
46
  - type: doi
42
47
  value: 10.5281/zenodo.22018869
@@ -2,6 +2,7 @@
2
2
  SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
3
  SPDX-License-Identifier: LGPL-3.0-or-later
4
4
  -->
5
+
5
6
  # CLAUDE.md
6
7
 
7
8
  This file provides guidance to Claude Code (claude.ai/code)
@@ -18,7 +19,7 @@ Guidance that applies to only part of the repo lives next to the code it governs
18
19
 
19
20
  - `stepup/core/CLAUDE.md`: process model, workflow-graph invariants, database conventions.
20
21
  - `tests/CLAUDE.md`: test layout and the integration-example suite.
21
- - `docs/CLAUDE.md`: regenerating the tutorial output.
22
+ - `docs/CLAUDE.md`: regenerating the tutorial output and social card.
22
23
  - `.claude/skills/release/SKILL.md`: the release procedure.
23
24
 
24
25
  ## Non-Negotiables
@@ -62,16 +63,22 @@ The following test command completes quickly as it skips the integration tests:
62
63
  pytest -k "not test_example"
63
64
  ```
64
65
 
65
- Always wrap the quick test run in a short timeout, e.g.:
66
-
67
- ```bash
68
- timeout 15 pytest -k "not test_example"
69
- ```
70
-
71
- 15 seconds is very generous for this selection.
72
- If a step crashes, the `client` test fixture can otherwise block indefinitely
73
- (it waits for the workflow to reach a state that never arrives),
74
- so a timeout prevents a runaway, hanging test process.
66
+ Every test is bounded by `pytest-timeout`, configured in `pyproject.toml`.
67
+ Setup, call and teardown of one test share a budget of 15 seconds,
68
+ which is very generous for this selection.
69
+ Without it, a test can hang forever instead of failing,
70
+ because the asynchronous RPC client takes no timeout
71
+ and the director waits for its steps without a deadline.
72
+ `tests/test_interrupt.py` and `tests/test_examples.py` raise the budget with a
73
+ module-level `timeout` marker, because their tests police their own deadlines.
74
+ Keep those markers above the deadline the module enforces itself,
75
+ so that a failure is reported by the assertion that knows what went wrong.
76
+
77
+ The budget is not a complete safety net.
78
+ `pytest-timeout` cancels the timer of a test as soon as pytest reports a failure,
79
+ which leaves the teardown of a test whose setup failed entirely unguarded.
80
+ A fixture that waits for a subprocess must therefore bound that wait itself,
81
+ as the `client` fixture does with `SHUTDOWN_TIMEOUT`.
75
82
 
76
83
  It may also be useful to run a small number of integration tests,
77
84
  to get a first quick feedback on the overall system:
@@ -0,0 +1,44 @@
1
+ <!--
2
+ SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
+ SPDX-License-Identifier: CC-BY-SA-4.0
4
+ -->
5
+
6
+ # Contributing to StepUp Core
7
+
8
+ Contributions are welcome, from bug reports and documentation fixes to new features.
9
+
10
+ ## Reporting Bugs and Requesting Features
11
+
12
+ Open an [issue](https://github.com/reproducible-reporting/stepup-core/issues).
13
+ For a bug, please include the StepUp version, the Python version, the operating system,
14
+ and a minimal example that reproduces the problem.
15
+
16
+ ## Setting up a Development Environment
17
+
18
+ The [developer notes](https://reproducible-reporting.github.io/stepup-core/development/)
19
+ explain how to install the development environment with
20
+ [uv](https://docs.astral.sh/uv/) and [pre-commit](https://pre-commit.com/),
21
+ how to run the tests, and how to build the documentation locally.
22
+
23
+ ## Making Changes
24
+
25
+ - Discuss larger changes in an issue before you start working on them,
26
+ so that you do not invest time in an approach that turns out not to fit.
27
+ - Install the pre-commit hooks with `pre-commit install`.
28
+ They format and lint every commit,
29
+ and the same checks run in continuous integration.
30
+ - Add tests for the behaviour you change, and make sure `pytest` passes.
31
+ - Wrap English text with [Semantic Line Breaks](https://sembr.org/),
32
+ because it keeps the diffs of prose small.
33
+ - Add an entry to `docs/changelog.md` when your change is visible to users.
34
+
35
+ ## Pull Requests
36
+
37
+ Open a pull request against the `main` branch
38
+ and describe what the change does and why it is needed.
39
+ The general conventions of the Reproducible Reporting organization are documented in
40
+ the [organization contributing guide](https://github.com/reproducible-reporting/.github/blob/main/CONTRIBUTING.md).
41
+
42
+ By contributing, you agree that your contributions are licensed under the
43
+ [LGPL-3.0-or-later](https://github.com/reproducible-reporting/stepup-core/blob/main/LICENSE)
44
+ license.
@@ -1,14 +1,15 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup
3
- Version: 4.0.0rc13
4
- Summary: StepUp Core provides the basic framework for the StepUp build tool
3
+ Version: 4.0.1
4
+ Summary: A dynamic Python build tool for reproducible workflows
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
7
7
  Project-URL: Documentation, https://reproducible-reporting.github.io/stepup-core/
8
8
  Project-URL: Issues, https://github.com/reproducible-reporting/stepup-core/issues
9
9
  Project-URL: Source, https://github.com/reproducible-reporting/stepup-core/
10
10
  Project-URL: Changelog, https://reproducible-reporting.github.io/stepup-core/changelog/
11
- Classifier: Development Status :: 4 - Beta
11
+ Keywords: asyncio,build automation,build tool,dynamic dependencies,incremental build,parallel,POSIX,Python,reproducibility,reproducible research,StepUp,terminal application,workflow,workflow management
12
+ Classifier: Development Status :: 5 - Production/Stable
12
13
  Classifier: Environment :: Console
13
14
  Classifier: Intended Audience :: Education
14
15
  Classifier: Intended Audience :: Science/Research
@@ -24,7 +25,7 @@ Classifier: Topic :: Software Development :: Build Tools
24
25
  Requires-Python: >=3.11
25
26
  Description-Content-Type: text/markdown
26
27
  License-File: LICENSE
27
- Requires-Dist: asyncinotify>=4.0.9
28
+ Requires-Dist: asyncinotify>=4.0.9; sys_platform == "linux"
28
29
  Requires-Dist: attrs>=23.1.0
29
30
  Requires-Dist: cattrs>=23.2.3
30
31
  Requires-Dist: jinja2>=3.1.6
@@ -32,20 +33,21 @@ Requires-Dist: parse>=1.19.1
32
33
  Requires-Dist: path>=16.14.0
33
34
  Requires-Dist: pyyaml>=6.0
34
35
  Requires-Dist: rich>=13.0.0
36
+ Provides-Extra: tests
37
+ Requires-Dist: matplotlib>=3.8.0; extra == "tests"
38
+ Requires-Dist: numpy>=1.26.0; extra == "tests"
39
+ Requires-Dist: pytest>=8.2.0; extra == "tests"
40
+ Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
41
+ Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
42
+ Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
35
43
  Provides-Extra: dev
36
- Requires-Dist: cffconvert; extra == "dev"
37
- Requires-Dist: matplotlib; extra == "dev"
38
- Requires-Dist: mike; extra == "dev"
44
+ Requires-Dist: stepup[tests]; extra == "dev"
39
45
  Requires-Dist: mkdocs; extra == "dev"
40
46
  Requires-Dist: mkdocstrings[python]; extra == "dev"
41
47
  Requires-Dist: mkdocs-macros-plugin; extra == "dev"
42
48
  Requires-Dist: mkdocs-material; extra == "dev"
43
- Requires-Dist: numpy; extra == "dev"
44
- Requires-Dist: pytest; extra == "dev"
45
- Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
46
- Requires-Dist: pytest-xdist; extra == "dev"
47
- Requires-Dist: psutil; extra == "dev"
48
49
  Requires-Dist: reuse; extra == "dev"
50
+ Requires-Dist: snipwise<2.0a0; extra == "dev"
49
51
  Requires-Dist: yappi; extra == "dev"
50
52
  Dynamic: license-file
51
53
 
@@ -53,7 +55,6 @@ Dynamic: license-file
53
55
  SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
54
56
  SPDX-License-Identifier: LGPL-3.0-or-later
55
57
  -->
56
- <!-- markdownlint-disable line-length -->
57
58
 
58
59
  # StepUp Core
59
60
 
@@ -66,6 +67,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
66
67
  ![LGPL-3 License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)
67
68
  [![CodeFactor](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core/badge)](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
68
69
 
70
+ <!-- snipwise.md BEGIN abstract -->
69
71
  StepUp is a dynamic build tool and a modern alternative to
70
72
  [Make](https://en.wikipedia.org/wiki/Make_(software)).
71
73
  Its defining feature is that workflow generation and execution are unified:
@@ -74,6 +76,7 @@ While the workflow is being executed, any step can add more steps and dependenci
74
76
  based on the outputs built so far.
75
77
  This makes StepUp ideal for builds
76
78
  where the full set of dependencies cannot be determined in advance.
79
+ <!-- snipwise.md END abstract -->
77
80
 
78
81
  StepUp Core provides the basic framework for StepUp, without any domain-specific features.
79
82
  Those live in extension packages:
@@ -94,7 +97,7 @@ for details.
94
97
 
95
98
  ## Quick Visual Impression
96
99
 
97
- [![asciicast](https://asciinema.org/a/718833.svg)](https://asciinema.org/a/718833)
100
+ [![asciicast](https://asciinema.org/a/pJXDBvCXT9ndHDuo.svg)](https://asciinema.org/a/pJXDBvCXT9ndHDuo)
98
101
 
99
102
  ## Documentation
100
103
 
@@ -104,7 +107,7 @@ is available at <https://reproducible-reporting.github.io/stepup-core>.
104
107
  ## License
105
108
 
106
109
  StepUp Core is distributed under the terms of the
107
- [LGPL-3.0-or-later](LICENSE) license.
110
+ [LGPL-3.0-or-later](https://github.com/reproducible-reporting/stepup-core/blob/main/LICENSE) license.
108
111
  Contributions are welcome;
109
112
  see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
110
113
  to get started.
@@ -2,7 +2,6 @@
2
2
  SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
3
  SPDX-License-Identifier: LGPL-3.0-or-later
4
4
  -->
5
- <!-- markdownlint-disable line-length -->
6
5
 
7
6
  # StepUp Core
8
7
 
@@ -15,6 +14,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
15
14
  ![LGPL-3 License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)
16
15
  [![CodeFactor](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core/badge)](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
17
16
 
17
+ <!-- snipwise.md BEGIN abstract -->
18
18
  StepUp is a dynamic build tool and a modern alternative to
19
19
  [Make](https://en.wikipedia.org/wiki/Make_(software)).
20
20
  Its defining feature is that workflow generation and execution are unified:
@@ -23,6 +23,7 @@ While the workflow is being executed, any step can add more steps and dependenci
23
23
  based on the outputs built so far.
24
24
  This makes StepUp ideal for builds
25
25
  where the full set of dependencies cannot be determined in advance.
26
+ <!-- snipwise.md END abstract -->
26
27
 
27
28
  StepUp Core provides the basic framework for StepUp, without any domain-specific features.
28
29
  Those live in extension packages:
@@ -43,7 +44,7 @@ for details.
43
44
 
44
45
  ## Quick Visual Impression
45
46
 
46
- [![asciicast](https://asciinema.org/a/718833.svg)](https://asciinema.org/a/718833)
47
+ [![asciicast](https://asciinema.org/a/pJXDBvCXT9ndHDuo.svg)](https://asciinema.org/a/pJXDBvCXT9ndHDuo)
47
48
 
48
49
  ## Documentation
49
50
 
@@ -53,7 +54,7 @@ is available at <https://reproducible-reporting.github.io/stepup-core>.
53
54
  ## License
54
55
 
55
56
  StepUp Core is distributed under the terms of the
56
- [LGPL-3.0-or-later](LICENSE) license.
57
+ [LGPL-3.0-or-later](https://github.com/reproducible-reporting/stepup-core/blob/main/LICENSE) license.
57
58
  Contributions are welcome;
58
59
  see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
59
60
  to get started.
@@ -2,6 +2,11 @@
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
3
  version = 1
4
4
 
5
+ [[annotations]]
6
+ path = ".zenodo.json"
7
+ SPDX-FileCopyrightText = "2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
8
+ SPDX-License-Identifier = "LGPL-3.0-or-later"
9
+
5
10
  [[annotations]]
6
11
  path = "tests/examples/*/**"
7
12
  SPDX-FileCopyrightText = "2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
@@ -34,6 +39,8 @@ SPDX-License-Identifier = "LGPL-3.0-or-later"
34
39
  [[annotations]]
35
40
  path = [
36
41
  "docs/logo.svg",
42
+ "docs/social-card.jpg",
43
+ "docs/social-card.svg",
37
44
  "stepup/core/logo.svg",
38
45
  ]
39
46
  SPDX-FileCopyrightText = "2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
@@ -1,7 +1,7 @@
1
1
  # SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
3
  [build-system]
4
- requires = ["setuptools>=65.0", "setuptools_scm>=8.1"]
4
+ requires = ["setuptools>=77.0", "setuptools_scm>=8.1"]
5
5
  build-backend = "setuptools.build_meta"
6
6
 
7
7
  [project]
@@ -9,12 +9,30 @@ name = "stepup"
9
9
  authors = [
10
10
  { name="Toon Verstraelen", email="toon.verstraelen@ugent.be" },
11
11
  ]
12
- description = "StepUp Core provides the basic framework for the StepUp build tool"
12
+ description = "A dynamic Python build tool for reproducible workflows"
13
13
  readme = "README.md"
14
14
  license = "LGPL-3.0-or-later"
15
15
  requires-python = ">=3.11"
16
+ keywords = [
17
+ # snipwise.md BEGIN keywords
18
+ "asyncio",
19
+ "build automation",
20
+ "build tool",
21
+ "dynamic dependencies",
22
+ "incremental build",
23
+ "parallel",
24
+ "POSIX",
25
+ "Python",
26
+ "reproducibility",
27
+ "reproducible research",
28
+ "StepUp",
29
+ "terminal application",
30
+ "workflow",
31
+ "workflow management",
32
+ # snipwise.md END keywords
33
+ ]
16
34
  classifiers = [
17
- "Development Status :: 4 - Beta",
35
+ "Development Status :: 5 - Production/Stable",
18
36
  "Environment :: Console",
19
37
  "Intended Audience :: Education",
20
38
  "Intended Audience :: Science/Research",
@@ -29,8 +47,7 @@ classifiers = [
29
47
  "Topic :: Software Development :: Build Tools",
30
48
  ]
31
49
  dependencies = [
32
- # Ensure changes to these dependencies are reflected in .github/requirements-old.txt
33
- "asyncinotify>=4.0.9",
50
+ "asyncinotify>=4.0.9 ; sys_platform == 'linux'",
34
51
  "attrs>=23.1.0",
35
52
  "cattrs>=23.2.3",
36
53
  "jinja2>=3.1.6",
@@ -42,20 +59,26 @@ dependencies = [
42
59
  dynamic = ["version"]
43
60
 
44
61
  [project.optional-dependencies]
62
+ # What the test suite needs, and nothing more.
63
+ # The pytest workflow installs this extra rather than `dev`,
64
+ # because it resolves the oldest supported versions of the direct dependencies,
65
+ # which is only meaningful for packages that the tests actually exercise.
66
+ tests = [
67
+ "matplotlib>=3.8.0",
68
+ "numpy>=1.26.0",
69
+ "pytest>=8.2.0",
70
+ "pytest-asyncio>=0.24",
71
+ "pytest-timeout>=2.3.1",
72
+ "pytest-xdist>=3.6.1",
73
+ ]
45
74
  dev = [
46
- "cffconvert",
47
- "matplotlib",
48
- "mike",
75
+ "stepup[tests]",
49
76
  "mkdocs",
50
77
  "mkdocstrings[python]",
51
78
  "mkdocs-macros-plugin",
52
79
  "mkdocs-material",
53
- "numpy",
54
- "pytest",
55
- "pytest-asyncio>=0.24",
56
- "pytest-xdist",
57
- "psutil",
58
80
  "reuse",
81
+ "snipwise<2.0a0",
59
82
  "yappi",
60
83
  ]
61
84
 
@@ -84,7 +107,6 @@ config = "stepup.core.config_tool:add_config_subcommand"
84
107
  shutdown = "stepup.core.interact:add_shutdown_subcommand"
85
108
  status = "stepup.core.status:add_status_subcommand"
86
109
 
87
-
88
110
  [tool.pytest.ini_options]
89
111
  addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
90
112
  asyncio_mode = "auto"
@@ -92,6 +114,7 @@ asyncio_default_fixture_loop_scope = "function"
92
114
  markers = [
93
115
  "requires_forkserver: mark test that requires STEPUP_BUILD_FORKSERVER != 0",
94
116
  ]
117
+ timeout = 15
95
118
 
96
119
  [tool.ruff]
97
120
  line-length = 100
@@ -128,3 +151,27 @@ packages = ["stepup.core"]
128
151
  [tool.setuptools_scm]
129
152
  version_scheme = "post-release"
130
153
  local_scheme = "no-local-version"
154
+
155
+ [tool.pymarkdown]
156
+ # Report unknown or mistyped settings instead of silently ignoring them.
157
+ mode.strict-config = true
158
+ # Skill and agent definitions carry YAML front matter,
159
+ # which pymarkdown parses as a setext heading unless this extension is on.
160
+ extensions.front-matter.enabled = true
161
+ # Rule md007 (ul-indent) expects a list nested in an ordered list item
162
+ # to be indented to the parent's content column, i.e. three spaces for a `1. ` item.
163
+ # Python-Markdown, and therefore MkDocs, only nests such a list when it is indented four spaces.
164
+ plugins.md007.enabled = false
165
+ plugins.md013.line_length = 100
166
+ plugins.md013.heading_line_length = 100
167
+ plugins.md013.code_block_line_length = 100
168
+ # The changelog repeats headings such as "Fixed" under every release,
169
+ # which is fine as long as they do not repeat under the same release.
170
+ plugins.md024.siblings_only = true
171
+ # The default list does not include `script`, which the documentation uses to embed asciinema.
172
+ plugins.md033.allowed_elements = "!--,![CDATA[,!DOCTYPE,script"
173
+ # Rule md041 (first-line-heading) counts the SPDX header in an HTML comment
174
+ # as content preceding the first heading.
175
+ plugins.md041.enabled = false
176
+ # The documentation mixes fenced and indented code blocks on purpose.
177
+ plugins.md046.enabled = false
@@ -0,0 +1,122 @@
1
+ <!--
2
+ SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
+ SPDX-License-Identifier: LGPL-3.0-or-later
4
+ -->
5
+
6
+ # Snipwise Configuration
7
+
8
+ Consult the full documentation at <https://reproducible-reporting.github.io/snipwise/>.
9
+
10
+ ```toml
11
+ # The Markdown files show the abstract as it is written below, links and all.
12
+ [[targets]]
13
+ patterns = ["README.md", "docs/index.md"]
14
+
15
+ # The summary of the Python package metadata, which is a single-line TOML string.
16
+ [[targets]]
17
+ patterns = ["pyproject.toml"]
18
+ scanner = "regex"
19
+ regex = '(?m)^description = "(?P<content>[^"]*)"$'
20
+ snippets = ["tagline"]
21
+ render = "{{ content | unwrap }}"
22
+
23
+ # The meta description that every documentation page carries.
24
+ [[targets]]
25
+ patterns = ["mkdocs.yaml"]
26
+ scanner = "regex"
27
+ regex = '(?m)^site_description: (?P<content>.*)$'
28
+ snippets = ["tagline"]
29
+ render = "{{ content | unwrap }}"
30
+
31
+ # The docstring of the package, which is a single-line string.
32
+ [[targets]]
33
+ patterns = ["stepup/core/__init__.py"]
34
+ scanner = "regex"
35
+ regex = '(?m)^"""(?P<content>[^"]*)"""$'
36
+ snippets = ["tagline"]
37
+ render = "{{ content | unwrap | suffix('.') }}"
38
+
39
+ # The CLI description.
40
+ [[targets]]
41
+ patterns = ["stepup/core/__main__.py"]
42
+ scanner = "regex"
43
+ regex = '(?m)^\s*description="(?P<content>[^"]*)\\n\\n"$'
44
+ snippets = ["tagline"]
45
+ render = "{{ content | unwrap | suffix('.') }}"
46
+
47
+ # The tagline in the social card, whose tspan carries the id that locates it.
48
+ [[targets]]
49
+ patterns = ["docs/social-card.svg"]
50
+ scanner = "regex"
51
+ regex = '<tspan\b[^>]*\bid="tagline"[^>]*>(?P<content>[^<]*)</tspan>'
52
+ snippets = ["tagline"]
53
+ render = "{{ content | unwrap }}"
54
+
55
+ # The keywords array of the Python package metadata.
56
+ [[targets]]
57
+ patterns = ["pyproject.toml"]
58
+ snippets = ["keywords"]
59
+ render = '''{{ content | prefix('"') | suffix('",') }}'''
60
+
61
+ # The abstract of the citation metadata, which is a folded YAML block scalar.
62
+ # The template terminates the last line, because the region includes its newline.
63
+ [[targets]]
64
+ patterns = ["CITATION.cff"]
65
+ scanner = "regex"
66
+ regex = '(?m)^abstract: >-\n(?P<content>(?:^ .*\n)+)'
67
+ snippets = ["abstract"]
68
+ render = "{{ content | plain | prefix(' ') }}\n"
69
+
70
+ # The keywords sequence of the citation metadata.
71
+ [[targets]]
72
+ patterns = ["CITATION.cff"]
73
+ snippets = ["keywords"]
74
+ render = "{{ content | prefix('- ') }}"
75
+
76
+ # The Zenodo metadata, which is JSON and therefore carries no markers.
77
+ [[targets]]
78
+ patterns = [".zenodo.json"]
79
+ scanner = "json"
80
+ insert = [
81
+ { snippet = "abstract", pointer = "/description", render = "{{ content | plain | unwrap }}" },
82
+ { snippet = "keywords", pointer = "/keywords", shape = "lines" },
83
+ ]
84
+ ```
85
+
86
+ ## `tagline`
87
+
88
+ ```text
89
+ A dynamic Python build tool for reproducible workflows
90
+ ```
91
+
92
+ ## `abstract`
93
+
94
+ ```markdown
95
+ StepUp is a dynamic build tool and a modern alternative to
96
+ [Make](https://en.wikipedia.org/wiki/Make_(software)).
97
+ Its defining feature is that workflow generation and execution are unified:
98
+ a `plan.py` script defines the initial build steps.
99
+ While the workflow is being executed, any step can add more steps and dependencies,
100
+ based on the outputs built so far.
101
+ This makes StepUp ideal for builds
102
+ where the full set of dependencies cannot be determined in advance.
103
+ ```
104
+
105
+ ## `keywords`
106
+
107
+ ```text
108
+ asyncio
109
+ build automation
110
+ build tool
111
+ dynamic dependencies
112
+ incremental build
113
+ parallel
114
+ POSIX
115
+ Python
116
+ reproducibility
117
+ reproducible research
118
+ StepUp
119
+ terminal application
120
+ workflow
121
+ workflow management
122
+ ```
@@ -2,6 +2,7 @@
2
2
  SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
3
3
  SPDX-License-Identifier: LGPL-3.0-or-later
4
4
  -->
5
+
5
6
  # Architecture of StepUp Core
6
7
 
7
8
  This file documents the design of `stepup/core/`:
@@ -317,12 +318,17 @@ and rescans its contents, so nothing created in the gap is missed.
317
318
 
318
319
  ### Removal
319
320
 
320
- `Workflow.to_be_deleted` holds files and directories alike,
321
+ The deletion queue holds files and directories alike,
321
322
  distinguished by a trailing separator on the key.
322
- `File.before_delete` marks the file's parent directory,
323
- `Step.before_delete` marks the step's working directory,
324
- and `revert_optional_steps` marks the parents of the outputs it flags,
325
- whose nodes stay in the graph and therefore never reach `before_delete`.
323
+ `Workflow.delete_detached` opens one for the duration of its own call
324
+ (reachable as `Workflow.to_be_deleted`, which raises at any other moment)
325
+ and returns it, so no queue outlives the cleanup pass that fills it.
326
+ `File.before_delete` marks the file's parent directory
327
+ and `Step.before_delete` marks the step's working directory.
328
+ `revert_optional_steps` builds a queue of its own and returns it,
329
+ because the nodes of the outputs it flags stay in the graph
330
+ and therefore never reach `before_delete`.
331
+ `Builder.finalize` merges the two queues and hands them to `remove_deletable_files`.
326
332
  A directory is marked whatever the state of the file that named it,
327
333
  which is what catches an output the user removed by hand:
328
334
  the file itself is then no longer in `to_be_deleted`, but its directory still is.
@@ -332,6 +338,9 @@ the file itself is then no longer in `to_be_deleted`, but its directory still is
332
338
  and then walks up to the root doing the same.
333
339
  Emptiness is the only safeguard here,
334
340
  so marking a directory too eagerly is harmless by construction.
341
+ The one exception is the project root and its ancestors,
342
+ which `mark_dir_to_be_deleted` refuses to queue and the upward walk stops at:
343
+ a step may build outside the root, but the directories above it are not StepUp's to remove.
335
344
 
336
345
  ## File Path Considerations
337
346
 
@@ -1,5 +1,5 @@
1
1
  # SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
2
2
  # SPDX-License-Identifier: LGPL-3.0-or-later
3
- """StepUp Core package."""
3
+ """A dynamic Python build tool for reproducible workflows."""
4
4
 
5
5
  __all__ = ()