stepup 4.0.0rc13__tar.gz → 4.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepup-4.0.1/.zenodo.json +64 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/CITATION.cff +13 -8
- {stepup-4.0.0rc13 → stepup-4.0.1}/CLAUDE.md +18 -11
- stepup-4.0.1/CONTRIBUTING.md +44 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/PKG-INFO +18 -15
- {stepup-4.0.0rc13 → stepup-4.0.1}/README.md +4 -3
- {stepup-4.0.0rc13 → stepup-4.0.1}/REUSE.toml +7 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/pyproject.toml +61 -14
- stepup-4.0.1/snipwise.md +122 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/CLAUDE.md +14 -5
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/__init__.py +1 -1
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/__main__.py +4 -2
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/builder.py +72 -17
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/director.py +8 -7
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/enums.py +37 -11
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/executor.py +20 -28
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/finalize.py +31 -15
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/hash.py +51 -42
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/hash_queue.py +59 -28
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/render_jinja.py +7 -5
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/startup.py +23 -31
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/step.py +0 -9
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/watcher.py +14 -12
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/workflow.py +395 -210
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/PKG-INFO +18 -15
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/SOURCES.txt +5 -2
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/requires.txt +13 -9
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/scm_file_list.json +9 -5
- stepup-4.0.1/stepup.egg-info/scm_version.json +8 -0
- stepup-4.0.1/tools/make_stable_redirects.py +119 -0
- stepup-4.0.0rc13/.claude/skills/release/SKILL.md +0 -14
- stepup-4.0.0rc13/stepup.egg-info/scm_version.json +0 -8
- {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSE +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSES/CC-BY-SA-4.0.txt +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSES/LGPL-3.0-or-later.txt +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/MANIFEST.in +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/setup.cfg +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/api.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/asyncio.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/browse.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/call.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/cattrs.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/cgroups.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/clean.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/config_loader.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/config_tool.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/constants.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/exceptions.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/extapi.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/file.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/interact.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/job.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/logo.svg +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/nglob.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/outcome.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/path.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/pending.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/pytest.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/reporter.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/rpc.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/run.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/scheduler.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/script.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/sqlite3.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/static_tree.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/status.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/stepinfo.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/tool.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/tracebacks.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/trellis.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/tui.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/usage.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup/core/utils.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/dependency_links.txt +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/entry_points.txt +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/stepup.egg-info/top_level.txt +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/tools/.gitignore +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/tools/analyze_perf.py +0 -0
- {stepup-4.0.0rc13 → stepup-4.0.1}/tools/leaves.py +0 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
{
|
|
2
|
+
"upload_type": "software",
|
|
3
|
+
"title": "StepUp Core",
|
|
4
|
+
"description": "StepUp is a dynamic build tool and a modern alternative to Make. Its defining feature is that workflow generation and execution are unified: a plan.py script defines the initial build steps. While the workflow is being executed, any step can add more steps and dependencies, based on the outputs built so far. This makes StepUp ideal for builds where the full set of dependencies cannot be determined in advance.",
|
|
5
|
+
"creators": [
|
|
6
|
+
{
|
|
7
|
+
"name": "Verstraelen, Toon",
|
|
8
|
+
"orcid": "0000-0001-9288-5608",
|
|
9
|
+
"affiliation": "Center for Molecular Modeling (CMM), Ghent University"
|
|
10
|
+
}
|
|
11
|
+
],
|
|
12
|
+
"license": "lgpl-3.0-or-later",
|
|
13
|
+
"access_right": "open",
|
|
14
|
+
"communities": [
|
|
15
|
+
{
|
|
16
|
+
"identifier": "ugent"
|
|
17
|
+
}
|
|
18
|
+
],
|
|
19
|
+
"language": "eng",
|
|
20
|
+
"keywords": [
|
|
21
|
+
"asyncio",
|
|
22
|
+
"build automation",
|
|
23
|
+
"build tool",
|
|
24
|
+
"dynamic dependencies",
|
|
25
|
+
"incremental build",
|
|
26
|
+
"parallel",
|
|
27
|
+
"POSIX",
|
|
28
|
+
"Python",
|
|
29
|
+
"reproducibility",
|
|
30
|
+
"reproducible research",
|
|
31
|
+
"StepUp",
|
|
32
|
+
"terminal application",
|
|
33
|
+
"workflow",
|
|
34
|
+
"workflow management"
|
|
35
|
+
],
|
|
36
|
+
"related_identifiers": [
|
|
37
|
+
{
|
|
38
|
+
"identifier": "https://reproducible-reporting.github.io/stepup-core",
|
|
39
|
+
"relation": "isDocumentedBy",
|
|
40
|
+
"resource_type": "publication-softwaredocumentation"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"identifier": "10.5281/zenodo.22024042",
|
|
44
|
+
"relation": "isRequiredBy",
|
|
45
|
+
"resource_type": "software"
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
"identifier": "10.5281/zenodo.22024209",
|
|
49
|
+
"relation": "isRequiredBy",
|
|
50
|
+
"resource_type": "software"
|
|
51
|
+
}
|
|
52
|
+
],
|
|
53
|
+
"custom": {
|
|
54
|
+
"code:codeRepository": "https://github.com/reproducible-reporting/stepup-core/",
|
|
55
|
+
"code:programmingLanguage": [
|
|
56
|
+
{
|
|
57
|
+
"id": "python"
|
|
58
|
+
}
|
|
59
|
+
],
|
|
60
|
+
"code:developmentStatus": {
|
|
61
|
+
"id": "active"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
@@ -6,7 +6,8 @@ title: StepUp Core
|
|
|
6
6
|
type: software
|
|
7
7
|
url: https://reproducible-reporting.github.io/stepup-core
|
|
8
8
|
abstract: >-
|
|
9
|
-
StepUp is a dynamic build tool and a modern alternative to
|
|
9
|
+
StepUp is a dynamic build tool and a modern alternative to
|
|
10
|
+
Make.
|
|
10
11
|
Its defining feature is that workflow generation and execution are unified:
|
|
11
12
|
a plan.py script defines the initial build steps.
|
|
12
13
|
While the workflow is being executed, any step can add more steps and dependencies,
|
|
@@ -22,21 +23,25 @@ authors:
|
|
|
22
23
|
post-code: 9052
|
|
23
24
|
city: Zwijnaarde
|
|
24
25
|
country: BE
|
|
26
|
+
license: LGPL-3.0-or-later
|
|
27
|
+
repository-code: https://github.com/reproducible-reporting/stepup-core
|
|
25
28
|
keywords:
|
|
29
|
+
# snipwise.md BEGIN keywords
|
|
26
30
|
- asyncio
|
|
27
|
-
- build tool
|
|
28
31
|
- build automation
|
|
29
|
-
-
|
|
32
|
+
- build tool
|
|
30
33
|
- dynamic dependencies
|
|
31
34
|
- incremental build
|
|
32
|
-
-
|
|
35
|
+
- parallel
|
|
36
|
+
- POSIX
|
|
37
|
+
- Python
|
|
33
38
|
- reproducibility
|
|
39
|
+
- reproducible research
|
|
40
|
+
- StepUp
|
|
34
41
|
- terminal application
|
|
35
42
|
- workflow
|
|
36
|
-
-
|
|
37
|
-
|
|
38
|
-
license: LGPL-3.0-or-later
|
|
39
|
-
repository-code: https://github.com/reproducible-reporting/stepup-core
|
|
43
|
+
- workflow management
|
|
44
|
+
# snipwise.md END keywords
|
|
40
45
|
identifiers:
|
|
41
46
|
- type: doi
|
|
42
47
|
value: 10.5281/zenodo.22018869
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
3
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
4
|
-->
|
|
5
|
+
|
|
5
6
|
# CLAUDE.md
|
|
6
7
|
|
|
7
8
|
This file provides guidance to Claude Code (claude.ai/code)
|
|
@@ -18,7 +19,7 @@ Guidance that applies to only part of the repo lives next to the code it governs
|
|
|
18
19
|
|
|
19
20
|
- `stepup/core/CLAUDE.md`: process model, workflow-graph invariants, database conventions.
|
|
20
21
|
- `tests/CLAUDE.md`: test layout and the integration-example suite.
|
|
21
|
-
- `docs/CLAUDE.md`: regenerating the tutorial output.
|
|
22
|
+
- `docs/CLAUDE.md`: regenerating the tutorial output and social card.
|
|
22
23
|
- `.claude/skills/release/SKILL.md`: the release procedure.
|
|
23
24
|
|
|
24
25
|
## Non-Negotiables
|
|
@@ -62,16 +63,22 @@ The following test command completes quickly as it skips the integration tests:
|
|
|
62
63
|
pytest -k "not test_example"
|
|
63
64
|
```
|
|
64
65
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
so a
|
|
66
|
+
Every test is bounded by `pytest-timeout`, configured in `pyproject.toml`.
|
|
67
|
+
Setup, call and teardown of one test share a budget of 15 seconds,
|
|
68
|
+
which is very generous for this selection.
|
|
69
|
+
Without it, a test can hang forever instead of failing,
|
|
70
|
+
because the asynchronous RPC client takes no timeout
|
|
71
|
+
and the director waits for its steps without a deadline.
|
|
72
|
+
`tests/test_interrupt.py` and `tests/test_examples.py` raise the budget with a
|
|
73
|
+
module-level `timeout` marker, because their tests police their own deadlines.
|
|
74
|
+
Keep those markers above the deadline the module enforces itself,
|
|
75
|
+
so that a failure is reported by the assertion that knows what went wrong.
|
|
76
|
+
|
|
77
|
+
The budget is not a complete safety net.
|
|
78
|
+
`pytest-timeout` cancels the timer of a test as soon as pytest reports a failure,
|
|
79
|
+
which leaves the teardown of a test whose setup failed entirely unguarded.
|
|
80
|
+
A fixture that waits for a subprocess must therefore bound that wait itself,
|
|
81
|
+
as the `client` fixture does with `SHUTDOWN_TIMEOUT`.
|
|
75
82
|
|
|
76
83
|
It may also be useful to run a small number of integration tests,
|
|
77
84
|
to get a first quick feedback on the overall system:
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
|
+
SPDX-License-Identifier: CC-BY-SA-4.0
|
|
4
|
+
-->
|
|
5
|
+
|
|
6
|
+
# Contributing to StepUp Core
|
|
7
|
+
|
|
8
|
+
Contributions are welcome, from bug reports and documentation fixes to new features.
|
|
9
|
+
|
|
10
|
+
## Reporting Bugs and Requesting Features
|
|
11
|
+
|
|
12
|
+
Open an [issue](https://github.com/reproducible-reporting/stepup-core/issues).
|
|
13
|
+
For a bug, please include the StepUp version, the Python version, the operating system,
|
|
14
|
+
and a minimal example that reproduces the problem.
|
|
15
|
+
|
|
16
|
+
## Setting up a Development Environment
|
|
17
|
+
|
|
18
|
+
The [developer notes](https://reproducible-reporting.github.io/stepup-core/development/)
|
|
19
|
+
explain how to install the development environment with
|
|
20
|
+
[uv](https://docs.astral.sh/uv/) and [pre-commit](https://pre-commit.com/),
|
|
21
|
+
how to run the tests, and how to build the documentation locally.
|
|
22
|
+
|
|
23
|
+
## Making Changes
|
|
24
|
+
|
|
25
|
+
- Discuss larger changes in an issue before you start working on them,
|
|
26
|
+
so that you do not invest time in an approach that turns out not to fit.
|
|
27
|
+
- Install the pre-commit hooks with `pre-commit install`.
|
|
28
|
+
They format and lint every commit,
|
|
29
|
+
and the same checks run in continuous integration.
|
|
30
|
+
- Add tests for the behaviour you change, and make sure `pytest` passes.
|
|
31
|
+
- Wrap English text with [Semantic Line Breaks](https://sembr.org/),
|
|
32
|
+
because it keeps the diffs of prose small.
|
|
33
|
+
- Add an entry to `docs/changelog.md` when your change is visible to users.
|
|
34
|
+
|
|
35
|
+
## Pull Requests
|
|
36
|
+
|
|
37
|
+
Open a pull request against the `main` branch
|
|
38
|
+
and describe what the change does and why it is needed.
|
|
39
|
+
The general conventions of the Reproducible Reporting organization are documented in
|
|
40
|
+
the [organization contributing guide](https://github.com/reproducible-reporting/.github/blob/main/CONTRIBUTING.md).
|
|
41
|
+
|
|
42
|
+
By contributing, you agree that your contributions are licensed under the
|
|
43
|
+
[LGPL-3.0-or-later](https://github.com/reproducible-reporting/stepup-core/blob/main/LICENSE)
|
|
44
|
+
license.
|
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: stepup
|
|
3
|
-
Version: 4.0.
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 4.0.1
|
|
4
|
+
Summary: A dynamic Python build tool for reproducible workflows
|
|
5
5
|
Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
|
|
6
6
|
License-Expression: LGPL-3.0-or-later
|
|
7
7
|
Project-URL: Documentation, https://reproducible-reporting.github.io/stepup-core/
|
|
8
8
|
Project-URL: Issues, https://github.com/reproducible-reporting/stepup-core/issues
|
|
9
9
|
Project-URL: Source, https://github.com/reproducible-reporting/stepup-core/
|
|
10
10
|
Project-URL: Changelog, https://reproducible-reporting.github.io/stepup-core/changelog/
|
|
11
|
-
|
|
11
|
+
Keywords: asyncio,build automation,build tool,dynamic dependencies,incremental build,parallel,POSIX,Python,reproducibility,reproducible research,StepUp,terminal application,workflow,workflow management
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
13
|
Classifier: Environment :: Console
|
|
13
14
|
Classifier: Intended Audience :: Education
|
|
14
15
|
Classifier: Intended Audience :: Science/Research
|
|
@@ -24,7 +25,7 @@ Classifier: Topic :: Software Development :: Build Tools
|
|
|
24
25
|
Requires-Python: >=3.11
|
|
25
26
|
Description-Content-Type: text/markdown
|
|
26
27
|
License-File: LICENSE
|
|
27
|
-
Requires-Dist: asyncinotify>=4.0.9
|
|
28
|
+
Requires-Dist: asyncinotify>=4.0.9; sys_platform == "linux"
|
|
28
29
|
Requires-Dist: attrs>=23.1.0
|
|
29
30
|
Requires-Dist: cattrs>=23.2.3
|
|
30
31
|
Requires-Dist: jinja2>=3.1.6
|
|
@@ -32,20 +33,21 @@ Requires-Dist: parse>=1.19.1
|
|
|
32
33
|
Requires-Dist: path>=16.14.0
|
|
33
34
|
Requires-Dist: pyyaml>=6.0
|
|
34
35
|
Requires-Dist: rich>=13.0.0
|
|
36
|
+
Provides-Extra: tests
|
|
37
|
+
Requires-Dist: matplotlib>=3.8.0; extra == "tests"
|
|
38
|
+
Requires-Dist: numpy>=1.26.0; extra == "tests"
|
|
39
|
+
Requires-Dist: pytest>=8.2.0; extra == "tests"
|
|
40
|
+
Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
|
|
41
|
+
Requires-Dist: pytest-timeout>=2.3.1; extra == "tests"
|
|
42
|
+
Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
|
|
35
43
|
Provides-Extra: dev
|
|
36
|
-
Requires-Dist:
|
|
37
|
-
Requires-Dist: matplotlib; extra == "dev"
|
|
38
|
-
Requires-Dist: mike; extra == "dev"
|
|
44
|
+
Requires-Dist: stepup[tests]; extra == "dev"
|
|
39
45
|
Requires-Dist: mkdocs; extra == "dev"
|
|
40
46
|
Requires-Dist: mkdocstrings[python]; extra == "dev"
|
|
41
47
|
Requires-Dist: mkdocs-macros-plugin; extra == "dev"
|
|
42
48
|
Requires-Dist: mkdocs-material; extra == "dev"
|
|
43
|
-
Requires-Dist: numpy; extra == "dev"
|
|
44
|
-
Requires-Dist: pytest; extra == "dev"
|
|
45
|
-
Requires-Dist: pytest-asyncio>=0.24; extra == "dev"
|
|
46
|
-
Requires-Dist: pytest-xdist; extra == "dev"
|
|
47
|
-
Requires-Dist: psutil; extra == "dev"
|
|
48
49
|
Requires-Dist: reuse; extra == "dev"
|
|
50
|
+
Requires-Dist: snipwise<2.0a0; extra == "dev"
|
|
49
51
|
Requires-Dist: yappi; extra == "dev"
|
|
50
52
|
Dynamic: license-file
|
|
51
53
|
|
|
@@ -53,7 +55,6 @@ Dynamic: license-file
|
|
|
53
55
|
SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
54
56
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
55
57
|
-->
|
|
56
|
-
<!-- markdownlint-disable line-length -->
|
|
57
58
|
|
|
58
59
|
# StepUp Core
|
|
59
60
|
|
|
@@ -66,6 +67,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|
|
66
67
|

|
|
67
68
|
[](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
|
|
68
69
|
|
|
70
|
+
<!-- snipwise.md BEGIN abstract -->
|
|
69
71
|
StepUp is a dynamic build tool and a modern alternative to
|
|
70
72
|
[Make](https://en.wikipedia.org/wiki/Make_(software)).
|
|
71
73
|
Its defining feature is that workflow generation and execution are unified:
|
|
@@ -74,6 +76,7 @@ While the workflow is being executed, any step can add more steps and dependenci
|
|
|
74
76
|
based on the outputs built so far.
|
|
75
77
|
This makes StepUp ideal for builds
|
|
76
78
|
where the full set of dependencies cannot be determined in advance.
|
|
79
|
+
<!-- snipwise.md END abstract -->
|
|
77
80
|
|
|
78
81
|
StepUp Core provides the basic framework for StepUp, without any domain-specific features.
|
|
79
82
|
Those live in extension packages:
|
|
@@ -94,7 +97,7 @@ for details.
|
|
|
94
97
|
|
|
95
98
|
## Quick Visual Impression
|
|
96
99
|
|
|
97
|
-
[](https://asciinema.org/a/pJXDBvCXT9ndHDuo)
|
|
98
101
|
|
|
99
102
|
## Documentation
|
|
100
103
|
|
|
@@ -104,7 +107,7 @@ is available at <https://reproducible-reporting.github.io/stepup-core>.
|
|
|
104
107
|
## License
|
|
105
108
|
|
|
106
109
|
StepUp Core is distributed under the terms of the
|
|
107
|
-
[LGPL-3.0-or-later](LICENSE) license.
|
|
110
|
+
[LGPL-3.0-or-later](https://github.com/reproducible-reporting/stepup-core/blob/main/LICENSE) license.
|
|
108
111
|
Contributions are welcome;
|
|
109
112
|
see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
|
|
110
113
|
to get started.
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
3
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
4
|
-->
|
|
5
|
-
<!-- markdownlint-disable line-length -->
|
|
6
5
|
|
|
7
6
|
# StepUp Core
|
|
8
7
|
|
|
@@ -15,6 +14,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
|
|
|
15
14
|

|
|
16
15
|
[](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
|
|
17
16
|
|
|
17
|
+
<!-- snipwise.md BEGIN abstract -->
|
|
18
18
|
StepUp is a dynamic build tool and a modern alternative to
|
|
19
19
|
[Make](https://en.wikipedia.org/wiki/Make_(software)).
|
|
20
20
|
Its defining feature is that workflow generation and execution are unified:
|
|
@@ -23,6 +23,7 @@ While the workflow is being executed, any step can add more steps and dependenci
|
|
|
23
23
|
based on the outputs built so far.
|
|
24
24
|
This makes StepUp ideal for builds
|
|
25
25
|
where the full set of dependencies cannot be determined in advance.
|
|
26
|
+
<!-- snipwise.md END abstract -->
|
|
26
27
|
|
|
27
28
|
StepUp Core provides the basic framework for StepUp, without any domain-specific features.
|
|
28
29
|
Those live in extension packages:
|
|
@@ -43,7 +44,7 @@ for details.
|
|
|
43
44
|
|
|
44
45
|
## Quick Visual Impression
|
|
45
46
|
|
|
46
|
-
[](https://asciinema.org/a/pJXDBvCXT9ndHDuo)
|
|
47
48
|
|
|
48
49
|
## Documentation
|
|
49
50
|
|
|
@@ -53,7 +54,7 @@ is available at <https://reproducible-reporting.github.io/stepup-core>.
|
|
|
53
54
|
## License
|
|
54
55
|
|
|
55
56
|
StepUp Core is distributed under the terms of the
|
|
56
|
-
[LGPL-3.0-or-later](LICENSE) license.
|
|
57
|
+
[LGPL-3.0-or-later](https://github.com/reproducible-reporting/stepup-core/blob/main/LICENSE) license.
|
|
57
58
|
Contributions are welcome;
|
|
58
59
|
see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
|
|
59
60
|
to get started.
|
|
@@ -2,6 +2,11 @@
|
|
|
2
2
|
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
3
3
|
version = 1
|
|
4
4
|
|
|
5
|
+
[[annotations]]
|
|
6
|
+
path = ".zenodo.json"
|
|
7
|
+
SPDX-FileCopyrightText = "2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
8
|
+
SPDX-License-Identifier = "LGPL-3.0-or-later"
|
|
9
|
+
|
|
5
10
|
[[annotations]]
|
|
6
11
|
path = "tests/examples/*/**"
|
|
7
12
|
SPDX-FileCopyrightText = "2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
@@ -34,6 +39,8 @@ SPDX-License-Identifier = "LGPL-3.0-or-later"
|
|
|
34
39
|
[[annotations]]
|
|
35
40
|
path = [
|
|
36
41
|
"docs/logo.svg",
|
|
42
|
+
"docs/social-card.jpg",
|
|
43
|
+
"docs/social-card.svg",
|
|
37
44
|
"stepup/core/logo.svg",
|
|
38
45
|
]
|
|
39
46
|
SPDX-FileCopyrightText = "2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>"
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
2
2
|
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
3
3
|
[build-system]
|
|
4
|
-
requires = ["setuptools>=
|
|
4
|
+
requires = ["setuptools>=77.0", "setuptools_scm>=8.1"]
|
|
5
5
|
build-backend = "setuptools.build_meta"
|
|
6
6
|
|
|
7
7
|
[project]
|
|
@@ -9,12 +9,30 @@ name = "stepup"
|
|
|
9
9
|
authors = [
|
|
10
10
|
{ name="Toon Verstraelen", email="toon.verstraelen@ugent.be" },
|
|
11
11
|
]
|
|
12
|
-
description = "
|
|
12
|
+
description = "A dynamic Python build tool for reproducible workflows"
|
|
13
13
|
readme = "README.md"
|
|
14
14
|
license = "LGPL-3.0-or-later"
|
|
15
15
|
requires-python = ">=3.11"
|
|
16
|
+
keywords = [
|
|
17
|
+
# snipwise.md BEGIN keywords
|
|
18
|
+
"asyncio",
|
|
19
|
+
"build automation",
|
|
20
|
+
"build tool",
|
|
21
|
+
"dynamic dependencies",
|
|
22
|
+
"incremental build",
|
|
23
|
+
"parallel",
|
|
24
|
+
"POSIX",
|
|
25
|
+
"Python",
|
|
26
|
+
"reproducibility",
|
|
27
|
+
"reproducible research",
|
|
28
|
+
"StepUp",
|
|
29
|
+
"terminal application",
|
|
30
|
+
"workflow",
|
|
31
|
+
"workflow management",
|
|
32
|
+
# snipwise.md END keywords
|
|
33
|
+
]
|
|
16
34
|
classifiers = [
|
|
17
|
-
"Development Status ::
|
|
35
|
+
"Development Status :: 5 - Production/Stable",
|
|
18
36
|
"Environment :: Console",
|
|
19
37
|
"Intended Audience :: Education",
|
|
20
38
|
"Intended Audience :: Science/Research",
|
|
@@ -29,8 +47,7 @@ classifiers = [
|
|
|
29
47
|
"Topic :: Software Development :: Build Tools",
|
|
30
48
|
]
|
|
31
49
|
dependencies = [
|
|
32
|
-
|
|
33
|
-
"asyncinotify>=4.0.9",
|
|
50
|
+
"asyncinotify>=4.0.9 ; sys_platform == 'linux'",
|
|
34
51
|
"attrs>=23.1.0",
|
|
35
52
|
"cattrs>=23.2.3",
|
|
36
53
|
"jinja2>=3.1.6",
|
|
@@ -42,20 +59,26 @@ dependencies = [
|
|
|
42
59
|
dynamic = ["version"]
|
|
43
60
|
|
|
44
61
|
[project.optional-dependencies]
|
|
62
|
+
# What the test suite needs, and nothing more.
|
|
63
|
+
# The pytest workflow installs this extra rather than `dev`,
|
|
64
|
+
# because it resolves the oldest supported versions of the direct dependencies,
|
|
65
|
+
# which is only meaningful for packages that the tests actually exercise.
|
|
66
|
+
tests = [
|
|
67
|
+
"matplotlib>=3.8.0",
|
|
68
|
+
"numpy>=1.26.0",
|
|
69
|
+
"pytest>=8.2.0",
|
|
70
|
+
"pytest-asyncio>=0.24",
|
|
71
|
+
"pytest-timeout>=2.3.1",
|
|
72
|
+
"pytest-xdist>=3.6.1",
|
|
73
|
+
]
|
|
45
74
|
dev = [
|
|
46
|
-
"
|
|
47
|
-
"matplotlib",
|
|
48
|
-
"mike",
|
|
75
|
+
"stepup[tests]",
|
|
49
76
|
"mkdocs",
|
|
50
77
|
"mkdocstrings[python]",
|
|
51
78
|
"mkdocs-macros-plugin",
|
|
52
79
|
"mkdocs-material",
|
|
53
|
-
"numpy",
|
|
54
|
-
"pytest",
|
|
55
|
-
"pytest-asyncio>=0.24",
|
|
56
|
-
"pytest-xdist",
|
|
57
|
-
"psutil",
|
|
58
80
|
"reuse",
|
|
81
|
+
"snipwise<2.0a0",
|
|
59
82
|
"yappi",
|
|
60
83
|
]
|
|
61
84
|
|
|
@@ -84,7 +107,6 @@ config = "stepup.core.config_tool:add_config_subcommand"
|
|
|
84
107
|
shutdown = "stepup.core.interact:add_shutdown_subcommand"
|
|
85
108
|
status = "stepup.core.status:add_status_subcommand"
|
|
86
109
|
|
|
87
|
-
|
|
88
110
|
[tool.pytest.ini_options]
|
|
89
111
|
addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
|
|
90
112
|
asyncio_mode = "auto"
|
|
@@ -92,6 +114,7 @@ asyncio_default_fixture_loop_scope = "function"
|
|
|
92
114
|
markers = [
|
|
93
115
|
"requires_forkserver: mark test that requires STEPUP_BUILD_FORKSERVER != 0",
|
|
94
116
|
]
|
|
117
|
+
timeout = 15
|
|
95
118
|
|
|
96
119
|
[tool.ruff]
|
|
97
120
|
line-length = 100
|
|
@@ -128,3 +151,27 @@ packages = ["stepup.core"]
|
|
|
128
151
|
[tool.setuptools_scm]
|
|
129
152
|
version_scheme = "post-release"
|
|
130
153
|
local_scheme = "no-local-version"
|
|
154
|
+
|
|
155
|
+
[tool.pymarkdown]
|
|
156
|
+
# Report unknown or mistyped settings instead of silently ignoring them.
|
|
157
|
+
mode.strict-config = true
|
|
158
|
+
# Skill and agent definitions carry YAML front matter,
|
|
159
|
+
# which pymarkdown parses as a setext heading unless this extension is on.
|
|
160
|
+
extensions.front-matter.enabled = true
|
|
161
|
+
# Rule md007 (ul-indent) expects a list nested in an ordered list item
|
|
162
|
+
# to be indented to the parent's content column, i.e. three spaces for a `1. ` item.
|
|
163
|
+
# Python-Markdown, and therefore MkDocs, only nests such a list when it is indented four spaces.
|
|
164
|
+
plugins.md007.enabled = false
|
|
165
|
+
plugins.md013.line_length = 100
|
|
166
|
+
plugins.md013.heading_line_length = 100
|
|
167
|
+
plugins.md013.code_block_line_length = 100
|
|
168
|
+
# The changelog repeats headings such as "Fixed" under every release,
|
|
169
|
+
# which is fine as long as they do not repeat under the same release.
|
|
170
|
+
plugins.md024.siblings_only = true
|
|
171
|
+
# The default list does not include `script`, which the documentation uses to embed asciinema.
|
|
172
|
+
plugins.md033.allowed_elements = "!--,![CDATA[,!DOCTYPE,script"
|
|
173
|
+
# Rule md041 (first-line-heading) counts the SPDX header in an HTML comment
|
|
174
|
+
# as content preceding the first heading.
|
|
175
|
+
plugins.md041.enabled = false
|
|
176
|
+
# The documentation mixes fenced and indented code blocks on purpose.
|
|
177
|
+
plugins.md046.enabled = false
|
stepup-4.0.1/snipwise.md
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
|
+
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
|
+
-->
|
|
5
|
+
|
|
6
|
+
# Snipwise Configuration
|
|
7
|
+
|
|
8
|
+
Consult the full documentation at <https://reproducible-reporting.github.io/snipwise/>.
|
|
9
|
+
|
|
10
|
+
```toml
|
|
11
|
+
# The Markdown files show the abstract as it is written below, links and all.
|
|
12
|
+
[[targets]]
|
|
13
|
+
patterns = ["README.md", "docs/index.md"]
|
|
14
|
+
|
|
15
|
+
# The summary of the Python package metadata, which is a single-line TOML string.
|
|
16
|
+
[[targets]]
|
|
17
|
+
patterns = ["pyproject.toml"]
|
|
18
|
+
scanner = "regex"
|
|
19
|
+
regex = '(?m)^description = "(?P<content>[^"]*)"$'
|
|
20
|
+
snippets = ["tagline"]
|
|
21
|
+
render = "{{ content | unwrap }}"
|
|
22
|
+
|
|
23
|
+
# The meta description that every documentation page carries.
|
|
24
|
+
[[targets]]
|
|
25
|
+
patterns = ["mkdocs.yaml"]
|
|
26
|
+
scanner = "regex"
|
|
27
|
+
regex = '(?m)^site_description: (?P<content>.*)$'
|
|
28
|
+
snippets = ["tagline"]
|
|
29
|
+
render = "{{ content | unwrap }}"
|
|
30
|
+
|
|
31
|
+
# The docstring of the package, which is a single-line string.
|
|
32
|
+
[[targets]]
|
|
33
|
+
patterns = ["stepup/core/__init__.py"]
|
|
34
|
+
scanner = "regex"
|
|
35
|
+
regex = '(?m)^"""(?P<content>[^"]*)"""$'
|
|
36
|
+
snippets = ["tagline"]
|
|
37
|
+
render = "{{ content | unwrap | suffix('.') }}"
|
|
38
|
+
|
|
39
|
+
# The CLI description.
|
|
40
|
+
[[targets]]
|
|
41
|
+
patterns = ["stepup/core/__main__.py"]
|
|
42
|
+
scanner = "regex"
|
|
43
|
+
regex = '(?m)^\s*description="(?P<content>[^"]*)\\n\\n"$'
|
|
44
|
+
snippets = ["tagline"]
|
|
45
|
+
render = "{{ content | unwrap | suffix('.') }}"
|
|
46
|
+
|
|
47
|
+
# The tagline in the social card, whose tspan carries the id that locates it.
|
|
48
|
+
[[targets]]
|
|
49
|
+
patterns = ["docs/social-card.svg"]
|
|
50
|
+
scanner = "regex"
|
|
51
|
+
regex = '<tspan\b[^>]*\bid="tagline"[^>]*>(?P<content>[^<]*)</tspan>'
|
|
52
|
+
snippets = ["tagline"]
|
|
53
|
+
render = "{{ content | unwrap }}"
|
|
54
|
+
|
|
55
|
+
# The keywords array of the Python package metadata.
|
|
56
|
+
[[targets]]
|
|
57
|
+
patterns = ["pyproject.toml"]
|
|
58
|
+
snippets = ["keywords"]
|
|
59
|
+
render = '''{{ content | prefix('"') | suffix('",') }}'''
|
|
60
|
+
|
|
61
|
+
# The abstract of the citation metadata, which is a folded YAML block scalar.
|
|
62
|
+
# The template terminates the last line, because the region includes its newline.
|
|
63
|
+
[[targets]]
|
|
64
|
+
patterns = ["CITATION.cff"]
|
|
65
|
+
scanner = "regex"
|
|
66
|
+
regex = '(?m)^abstract: >-\n(?P<content>(?:^ .*\n)+)'
|
|
67
|
+
snippets = ["abstract"]
|
|
68
|
+
render = "{{ content | plain | prefix(' ') }}\n"
|
|
69
|
+
|
|
70
|
+
# The keywords sequence of the citation metadata.
|
|
71
|
+
[[targets]]
|
|
72
|
+
patterns = ["CITATION.cff"]
|
|
73
|
+
snippets = ["keywords"]
|
|
74
|
+
render = "{{ content | prefix('- ') }}"
|
|
75
|
+
|
|
76
|
+
# The Zenodo metadata, which is JSON and therefore carries no markers.
|
|
77
|
+
[[targets]]
|
|
78
|
+
patterns = [".zenodo.json"]
|
|
79
|
+
scanner = "json"
|
|
80
|
+
insert = [
|
|
81
|
+
{ snippet = "abstract", pointer = "/description", render = "{{ content | plain | unwrap }}" },
|
|
82
|
+
{ snippet = "keywords", pointer = "/keywords", shape = "lines" },
|
|
83
|
+
]
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## `tagline`
|
|
87
|
+
|
|
88
|
+
```text
|
|
89
|
+
A dynamic Python build tool for reproducible workflows
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## `abstract`
|
|
93
|
+
|
|
94
|
+
```markdown
|
|
95
|
+
StepUp is a dynamic build tool and a modern alternative to
|
|
96
|
+
[Make](https://en.wikipedia.org/wiki/Make_(software)).
|
|
97
|
+
Its defining feature is that workflow generation and execution are unified:
|
|
98
|
+
a `plan.py` script defines the initial build steps.
|
|
99
|
+
While the workflow is being executed, any step can add more steps and dependencies,
|
|
100
|
+
based on the outputs built so far.
|
|
101
|
+
This makes StepUp ideal for builds
|
|
102
|
+
where the full set of dependencies cannot be determined in advance.
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## `keywords`
|
|
106
|
+
|
|
107
|
+
```text
|
|
108
|
+
asyncio
|
|
109
|
+
build automation
|
|
110
|
+
build tool
|
|
111
|
+
dynamic dependencies
|
|
112
|
+
incremental build
|
|
113
|
+
parallel
|
|
114
|
+
POSIX
|
|
115
|
+
Python
|
|
116
|
+
reproducibility
|
|
117
|
+
reproducible research
|
|
118
|
+
StepUp
|
|
119
|
+
terminal application
|
|
120
|
+
workflow
|
|
121
|
+
workflow management
|
|
122
|
+
```
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
|
|
3
3
|
SPDX-License-Identifier: LGPL-3.0-or-later
|
|
4
4
|
-->
|
|
5
|
+
|
|
5
6
|
# Architecture of StepUp Core
|
|
6
7
|
|
|
7
8
|
This file documents the design of `stepup/core/`:
|
|
@@ -317,12 +318,17 @@ and rescans its contents, so nothing created in the gap is missed.
|
|
|
317
318
|
|
|
318
319
|
### Removal
|
|
319
320
|
|
|
320
|
-
|
|
321
|
+
The deletion queue holds files and directories alike,
|
|
321
322
|
distinguished by a trailing separator on the key.
|
|
322
|
-
`
|
|
323
|
-
`
|
|
324
|
-
and
|
|
325
|
-
|
|
323
|
+
`Workflow.delete_detached` opens one for the duration of its own call
|
|
324
|
+
(reachable as `Workflow.to_be_deleted`, which raises at any other moment)
|
|
325
|
+
and returns it, so no queue outlives the cleanup pass that fills it.
|
|
326
|
+
`File.before_delete` marks the file's parent directory
|
|
327
|
+
and `Step.before_delete` marks the step's working directory.
|
|
328
|
+
`revert_optional_steps` builds a queue of its own and returns it,
|
|
329
|
+
because the nodes of the outputs it flags stay in the graph
|
|
330
|
+
and therefore never reach `before_delete`.
|
|
331
|
+
`Builder.finalize` merges the two queues and hands them to `remove_deletable_files`.
|
|
326
332
|
A directory is marked whatever the state of the file that named it,
|
|
327
333
|
which is what catches an output the user removed by hand:
|
|
328
334
|
the file itself is then no longer in `to_be_deleted`, but its directory still is.
|
|
@@ -332,6 +338,9 @@ the file itself is then no longer in `to_be_deleted`, but its directory still is
|
|
|
332
338
|
and then walks up to the root doing the same.
|
|
333
339
|
Emptiness is the only safeguard here,
|
|
334
340
|
so marking a directory too eagerly is harmless by construction.
|
|
341
|
+
The one exception is the project root and its ancestors,
|
|
342
|
+
which `mark_dir_to_be_deleted` refuses to queue and the upward walk stops at:
|
|
343
|
+
a step may build outside the root, but the directories above it are not StepUp's to remove.
|
|
335
344
|
|
|
336
345
|
## File Path Considerations
|
|
337
346
|
|