stepup 4.0.2__tar.gz → 4.1.0rc1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. stepup-4.1.0rc1/.gitattributes +8 -0
  2. {stepup-4.0.2 → stepup-4.1.0rc1}/.zenodo.json +1 -1
  3. {stepup-4.0.2 → stepup-4.1.0rc1}/CITATION.cff +1 -1
  4. {stepup-4.0.2 → stepup-4.1.0rc1}/CLAUDE.md +80 -54
  5. {stepup-4.0.2 → stepup-4.1.0rc1}/CONTRIBUTING.md +2 -2
  6. {stepup-4.0.2 → stepup-4.1.0rc1}/MANIFEST.in +1 -1
  7. {stepup-4.0.2 → stepup-4.1.0rc1}/PKG-INFO +12 -9
  8. {stepup-4.0.2 → stepup-4.1.0rc1}/README.md +1 -1
  9. {stepup-4.0.2 → stepup-4.1.0rc1}/pyproject.toml +45 -24
  10. {stepup-4.0.2 → stepup-4.1.0rc1}/snipwise.md +3 -3
  11. stepup-4.1.0rc1/stepup/core/CLAUDE.md +259 -0
  12. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/__main__.py +4 -6
  13. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/api.py +158 -68
  14. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/call.py +4 -4
  15. stepup-4.1.0rc1/stepup/core/cattrs.py +94 -0
  16. stepup-4.1.0rc1/stepup/core/cli/CLAUDE.md +80 -0
  17. stepup-4.1.0rc1/stepup/core/cli/__init__.py +5 -0
  18. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/cli}/browse.py +8 -8
  19. stepup-4.0.2/stepup/core/tui.py → stepup-4.1.0rc1/stepup/core/cli/build.py +194 -510
  20. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/cli}/clean.py +8 -8
  21. stepup-4.0.2/stepup/core/config_tool.py → stepup-4.1.0rc1/stepup/core/cli/config.py +3 -3
  22. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/cli}/config_loader.py +4 -5
  23. stepup-4.1.0rc1/stepup/core/cli/director_log.py +162 -0
  24. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/cli}/interact.py +6 -8
  25. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/cli}/reporter.py +13 -126
  26. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/cli}/status.py +1 -1
  27. stepup-4.1.0rc1/stepup/core/cli/terminal.py +547 -0
  28. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/cli}/tool.py +4 -46
  29. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/constants.py +16 -1
  30. stepup-4.1.0rc1/stepup/core/director/CLAUDE.md +82 -0
  31. stepup-4.1.0rc1/stepup/core/director/__init__.py +8 -0
  32. stepup-4.1.0rc1/stepup/core/director/__main__.py +305 -0
  33. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/builder.py +8 -9
  34. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/executor.py +72 -25
  35. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/finalize.py +10 -12
  36. stepup-4.1.0rc1/stepup/core/director/handler.py +593 -0
  37. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/hash_queue.py +4 -3
  38. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/job.py +5 -5
  39. stepup-4.1.0rc1/stepup/core/director/reporter.py +154 -0
  40. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/scheduler.py +49 -39
  41. stepup-4.1.0rc1/stepup/core/director/serve.py +473 -0
  42. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/startup.py +6 -6
  43. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/usage.py +36 -11
  44. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/director}/watcher.py +22 -194
  45. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/enums.py +19 -1
  46. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/exceptions.py +17 -0
  47. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/extapi.py +68 -24
  48. stepup-4.1.0rc1/stepup/core/graph/CLAUDE.md +156 -0
  49. stepup-4.1.0rc1/stepup/core/graph/__init__.py +9 -0
  50. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/file.py +5 -6
  51. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/pending.py +3 -2
  52. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/sqlite3.py +6 -6
  53. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/step.py +9 -58
  54. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/trellis.py +1 -1
  55. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/workflow.py +21 -23
  56. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/hash.py +9 -2
  57. stepup-4.1.0rc1/stepup/core/launch/CLAUDE.md +125 -0
  58. stepup-4.1.0rc1/stepup/core/launch/__init__.py +8 -0
  59. stepup-4.1.0rc1/stepup/core/launch/capture.py +109 -0
  60. stepup-4.1.0rc1/stepup/core/launch/command.py +299 -0
  61. stepup-4.1.0rc1/stepup/core/launch/copy.py +360 -0
  62. stepup-4.1.0rc1/stepup/core/launch/forkserver.py +258 -0
  63. stepup-4.1.0rc1/stepup/core/launch/subprocess.py +225 -0
  64. stepup-4.1.0rc1/stepup/core/launch/worker.py +453 -0
  65. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/nglob.py +2 -1
  66. stepup-4.1.0rc1/stepup/core/path.py +610 -0
  67. stepup-4.1.0rc1/stepup/core/platform/CLAUDE.md +96 -0
  68. stepup-4.1.0rc1/stepup/core/platform/__init__.py +9 -0
  69. stepup-4.1.0rc1/stepup/core/platform/asyncio.py +331 -0
  70. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/platform}/cgroups.py +7 -8
  71. stepup-4.1.0rc1/stepup/core/platform/oscompat.py +86 -0
  72. stepup-4.1.0rc1/stepup/core/platform/popen.py +625 -0
  73. stepup-4.1.0rc1/stepup/core/platform/stdio.py +73 -0
  74. stepup-4.1.0rc1/stepup/core/platform/winapi.py +477 -0
  75. stepup-4.1.0rc1/stepup/core/pytest.py +775 -0
  76. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/render_jinja.py +3 -4
  77. stepup-4.1.0rc1/stepup/core/rpc/CLAUDE.md +43 -0
  78. stepup-4.1.0rc1/stepup/core/rpc/__init__.py +25 -0
  79. stepup-4.0.2/stepup/core/rpc.py → stepup-4.1.0rc1/stepup/core/rpc/client.py +89 -757
  80. stepup-4.1.0rc1/stepup/core/rpc/protocol.py +455 -0
  81. stepup-4.1.0rc1/stepup/core/rpc/server.py +612 -0
  82. stepup-4.1.0rc1/stepup/core/rpc/transport.py +192 -0
  83. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/script.py +2 -3
  84. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/stepinfo.py +3 -4
  85. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/tracebacks.py +6 -3
  86. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/utils.py +124 -128
  87. stepup-4.1.0rc1/stepup/core/watch/CLAUDE.md +52 -0
  88. stepup-4.1.0rc1/stepup/core/watch/__init__.py +5 -0
  89. stepup-4.1.0rc1/stepup/core/watch/backend.py +98 -0
  90. stepup-4.1.0rc1/stepup/core/watch/inotify.py +144 -0
  91. stepup-4.1.0rc1/stepup/core/watch/select.py +65 -0
  92. stepup-4.1.0rc1/stepup/core/watch/watchfiles.py +918 -0
  93. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup.egg-info/PKG-INFO +12 -9
  94. stepup-4.1.0rc1/stepup.egg-info/SOURCES.txt +119 -0
  95. stepup-4.1.0rc1/stepup.egg-info/entry_points.txt +18 -0
  96. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup.egg-info/requires.txt +9 -5
  97. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup.egg-info/scm_file_list.json +138 -69
  98. stepup-4.1.0rc1/stepup.egg-info/scm_version.json +8 -0
  99. stepup-4.1.0rc1/tools/README.md +24 -0
  100. {stepup-4.0.2/tools → stepup-4.1.0rc1/tools/docs}/make_stable_redirects.py +5 -5
  101. {stepup-4.0.2/tools → stepup-4.1.0rc1/tools/imports}/leaves.py +8 -6
  102. stepup-4.1.0rc1/tools/macci/README.md +149 -0
  103. stepup-4.1.0rc1/tools/macci/macci.sh +240 -0
  104. {stepup-4.0.2/tools → stepup-4.1.0rc1/tools/perf}/analyze_perf.py +4 -4
  105. stepup-4.1.0rc1/tools/winvm/README.md +156 -0
  106. stepup-4.1.0rc1/tools/winvm/oem/install.bat +10 -0
  107. stepup-4.1.0rc1/tools/winvm/oem/setup.ps1 +114 -0
  108. stepup-4.1.0rc1/tools/winvm/winvm.sh +307 -0
  109. stepup-4.0.2/stepup/core/CLAUDE.md +0 -394
  110. stepup-4.0.2/stepup/core/asyncio.py +0 -148
  111. stepup-4.0.2/stepup/core/cattrs.py +0 -63
  112. stepup-4.0.2/stepup/core/director.py +0 -1290
  113. stepup-4.0.2/stepup/core/path.py +0 -334
  114. stepup-4.0.2/stepup/core/pytest.py +0 -371
  115. stepup-4.0.2/stepup/core/run.py +0 -982
  116. stepup-4.0.2/stepup.egg-info/SOURCES.txt +0 -76
  117. stepup-4.0.2/stepup.egg-info/entry_points.txt +0 -18
  118. stepup-4.0.2/stepup.egg-info/scm_version.json +0 -8
  119. {stepup-4.0.2 → stepup-4.1.0rc1}/LICENSE +0 -0
  120. {stepup-4.0.2 → stepup-4.1.0rc1}/LICENSES/CC-BY-SA-4.0.txt +0 -0
  121. {stepup-4.0.2 → stepup-4.1.0rc1}/LICENSES/LGPL-3.0-or-later.txt +0 -0
  122. {stepup-4.0.2 → stepup-4.1.0rc1}/LICENSES/LicenseRef-StepUp-Logo-v1.txt +0 -0
  123. {stepup-4.0.2 → stepup-4.1.0rc1}/REUSE.toml +0 -0
  124. {stepup-4.0.2 → stepup-4.1.0rc1}/setup.cfg +0 -0
  125. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/__init__.py +0 -0
  126. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/outcome.py +0 -0
  127. {stepup-4.0.2/stepup/core → stepup-4.1.0rc1/stepup/core/graph}/static_tree.py +0 -0
  128. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup/core/logo.svg +0 -0
  129. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup.egg-info/dependency_links.txt +0 -0
  130. {stepup-4.0.2 → stepup-4.1.0rc1}/stepup.egg-info/top_level.txt +0 -0
  131. {stepup-4.0.2/tools → stepup-4.1.0rc1/tools/perf}/.gitignore +0 -0
@@ -0,0 +1,8 @@
1
+ # SPDX-FileCopyrightText: 2024 Toon Verstraelen <Toon.Verstraelen@UGent.be>
2
+ # SPDX-License-Identifier: LGPL-3.0-or-later
3
+
4
+ # Check out text files with LF line endings on every platform,
5
+ # also where Git converts them to CRLF by default, e.g. on GitHub's Windows runners.
6
+ # The examples in tests/ compare file digests and sizes with expected outputs,
7
+ # and the shell scripts in the repository do not tolerate a carriage return.
8
+ * text=auto eol=lf
@@ -21,10 +21,10 @@
21
21
  "asyncio",
22
22
  "build automation",
23
23
  "build tool",
24
+ "cross-platform",
24
25
  "dynamic dependencies",
25
26
  "incremental build",
26
27
  "parallel",
27
- "POSIX",
28
28
  "Python",
29
29
  "reproducibility",
30
30
  "reproducible research",
@@ -30,10 +30,10 @@ keywords:
30
30
  - asyncio
31
31
  - build automation
32
32
  - build tool
33
+ - cross-platform
33
34
  - dynamic dependencies
34
35
  - incremental build
35
36
  - parallel
36
- - POSIX
37
37
  - Python
38
38
  - reproducibility
39
39
  - reproducible research
@@ -13,14 +13,22 @@ when working with code in this repository.
13
13
  StepUp Core is a dynamic build tool implemented in Python.
14
14
  It runs a persistent **director** process that manages a workflow graph (stored in SQLite),
15
15
  dispatches jobs to a **step executor**,
16
- and reacts to file system changes (via inotify) to re-run only outdated steps.
16
+ and reacts to file system changes (via inotify on Linux, watchfiles elsewhere)
17
+ to re-run only outdated steps.
18
+
19
+ StepUp Core runs on Linux, macOS and Windows,
20
+ and what differs between them for users is stated in `docs/platforms.md`.
21
+ The rules below that exist for Windows are in [Platforms](#platforms).
17
22
 
18
23
  Guidance that applies to only part of the repo lives next to the code it governs:
19
24
 
20
- - `stepup/core/CLAUDE.md`: process model, workflow-graph invariants, database conventions.
21
- - `tests/CLAUDE.md`: test layout and the integration-example suite.
25
+ - `stepup/core/CLAUDE.md`: layering of the subpackages, process model, path conventions,
26
+ and the `__all__` import contract.
27
+ - `stepup/core/*/CLAUDE.md`: the design of each subpackage,
28
+ e.g. the workflow-graph invariants and database conventions in `stepup/core/graph/CLAUDE.md`.
29
+ - `tests/CLAUDE.md`: test layout, test timeouts and the integration-example suite.
22
30
  - `docs/CLAUDE.md`: regenerating the tutorial output and social card.
23
- - `.claude/skills/release/SKILL.md`: the release procedure.
31
+ - `docs/development.md`: the release procedure, under "How to Make a Release".
24
32
 
25
33
  ## Non-Negotiables
26
34
 
@@ -29,7 +37,7 @@ Guidance that applies to only part of the repo lives next to the code it governs
29
37
  - **The workflow graph must be a deterministic function of the source files and the
30
38
  plan/step code, never of scheduling order or job count.**
31
39
  Running the same project with `-j1` and `-j16` must produce the same graph.
32
- Rationale and consequences in `stepup/core/CLAUDE.md`.
40
+ Rationale and consequences in `stepup/core/graph/CLAUDE.md`.
33
41
 
34
42
  ## Commands
35
43
 
@@ -63,23 +71,6 @@ The following test command completes quickly as it skips the integration tests:
63
71
  pytest -k "not test_example"
64
72
  ```
65
73
 
66
- Every test is bounded by `pytest-timeout`, configured in `pyproject.toml`.
67
- Setup, call and teardown of one test share a budget of 15 seconds,
68
- which is very generous for this selection.
69
- Without it, a test can hang forever instead of failing,
70
- because the asynchronous RPC client takes no timeout
71
- and the director waits for its steps without a deadline.
72
- `tests/test_interrupt.py` and `tests/test_examples.py` raise the budget with a
73
- module-level `timeout` marker, because their tests police their own deadlines.
74
- Keep those markers above the deadline the module enforces itself,
75
- so that a failure is reported by the assertion that knows what went wrong.
76
-
77
- The budget is not a complete safety net.
78
- `pytest-timeout` cancels the timer of a test as soon as pytest reports a failure,
79
- which leaves the teardown of a test whose setup failed entirely unguarded.
80
- A fixture that waits for a subprocess must therefore bound that wait itself,
81
- as the `client` fixture does with `SHUTDOWN_TIMEOUT`.
82
-
83
74
  It may also be useful to run a small number of integration tests,
84
75
  to get a first quick feedback on the overall system:
85
76
 
@@ -102,8 +93,54 @@ system, which is a real cause of flaky failures in timing-sensitive examples
102
93
  non-dispatch).
103
94
  Run `pytest` invocations one after another instead.
104
95
 
96
+ ### Developer Tools
97
+
98
+ `tools/README.md` gives an overview of the developer scripts in `tools/`, grouped by theme.
99
+
100
+ To run the test suite on Windows, use the VM managed by `tools/winvm/winvm.sh`,
101
+ which is documented in `tools/winvm/README.md`.
102
+ When the VM exists, `tools/winvm/winvm.sh test ARGS` copies the working tree into it
103
+ and runs `uv run pytest ARGS` there.
104
+ The VM shares the CPUs of the host,
105
+ so the rule above against concurrent `pytest` runs applies to it as well.
106
+ A complete run of the examples takes about 10 minutes there.
107
+ Run the fast suite and a few examples first, and the complete example suite as a final check only.
108
+ Leave `create` and `destroy` to the user,
109
+ because they download or delete several gigabytes.
110
+
111
+ To run the test suite on macOS, use `tools/macci/macci.sh`,
112
+ which is documented in `tools/macci/README.md`.
113
+ `tools/macci/macci.sh test ARGS` pushes a snapshot of the working tree to the user's fork,
114
+ runs `uv run pytest ARGS` there on a GitHub-hosted macOS runner and prints the output.
115
+ Run it only when the user asks for it,
116
+ because the push publishes uncommitted changes in a public repository.
117
+ The runner does not use the CPUs of the host,
118
+ so it may run at the same time as a local `pytest` run.
119
+
105
120
  ## Coding Conventions
106
121
 
122
+ ### Platforms
123
+
124
+ Code in `stepup/` must work on Linux, macOS and Windows, and a change is not finished
125
+ when it passes on the platform it was written on.
126
+ The rules that matter most, with their reasons in `stepup/core/CLAUDE.md`
127
+ and `stepup/core/platform/CLAUDE.md`:
128
+
129
+ - **A path inside StepUp is a string with `/` as its only separator, also on Windows.**
130
+ Import `Path` from `stepup.core.path`, never write `os.sep`, and avoid `os.path`.
131
+ `ConventionTests.test_slash_paths` enforces this.
132
+ - **A command is text that StepUp never rewrites.**
133
+ Paths are embedded in a command with `shq()`.
134
+ - **Test for a feature, not for a platform.**
135
+ Use the predicates of `stepup/core/platform/oscompat.py`,
136
+ and the matching capability markers in tests (`tests/CLAUDE.md`).
137
+ - **Do not rely on the line endings or the permission bits of the checkout.**
138
+ `.gitattributes` makes every platform check out LF.
139
+ Text written by StepUp itself uses `\n`, see [Text Files](#text-files).
140
+ - **Keep `docs/platforms.md` true.**
141
+ It is the user-facing list of what differs per platform,
142
+ so a change to a limitation or to a feature that depends on the platform updates that page too.
143
+
107
144
  ### Semantic Line Breaks
108
145
 
109
146
  All English text in this repo is wrapped using **semantic line breaks**:
@@ -152,7 +189,8 @@ Stale prose is worse than no prose. When writing comments, docstrings, or other
152
189
  and it drifts out of sync silently.
153
190
  - **Repetitive and duplicate comments.**
154
191
  If a remark is repeated in multiple places, it will rot in one place when updated in another.
155
- Factor out the common remark into a single function or class docstring, or a Markdown file in `docs/`.
192
+ Factor out the common remark into a single function or class docstring,
193
+ or a Markdown file in `docs/`.
156
194
  - **Timeless phrasing for point-in-time claims.** An empirical observation about an external
157
195
  tool or environment (e.g. "SQLite's planner never picks this index") can stop being true
158
196
  after a version upgrade, with nothing to flag the comment as outdated.
@@ -166,6 +204,17 @@ be resolved by restructuring the code,
166
204
  because the project's rule set already excludes rules that would fire spuriously
167
205
  in this codebase.
168
206
 
207
+ ### Text Files
208
+
209
+ Open every text file with `encoding="utf-8"`, and write text files with `newline="\n"` as well,
210
+ so that files are identical on all platforms,
211
+ including Windows with Python 3.14 and older, where the default is the ANSI code page.
212
+ CSV files are the exception: they are opened with `newline=""`, as the `csv` module requires.
213
+ Ruff enforces the encoding with `PLW1514`,
214
+ but only for `open()` and `pathlib`, not for the methods of `path.Path`.
215
+ Their `read_text()` and `write_text()` need `encoding="utf-8"` too,
216
+ while `write_text()` already writes `\n` line endings by default.
217
+
169
218
  ### Docstrings
170
219
 
171
220
  Use **NumPy-style** sections (`Parameters`, `Returns`, `Raises`, ...)
@@ -212,35 +261,6 @@ Section headings (`##`, `###`, ...) use **Title Case**
212
261
  coordinating conjunctions, and prepositions regardless of length, e.g. "from", "with").
213
262
  Inline code spans (e.g. `` `run()` ``) keep their own casing and are never title-cased.
214
263
 
215
- ### `__all__`
216
-
217
- Wildcard imports are banned (ruff `F403`), so `__all__` does not describe a star-import
218
- surface here. It is the module's **import contract**: the names that code outside the module
219
- is meant to import.
220
-
221
- - Every module in `stepup/core/` declares `__all__`, placed directly after the imports and
222
- before `logger`. It is a tuple of string literals, sorted (enforced by ruff `RUF022`).
223
- - List a name when it is imported by another `stepup` module, a downstream extension package,
224
- a user's `plan.py`, or a `pyproject.toml` entry point (e.g. `build_subcommand`).
225
- - Do not list module-internal names, even when they lack a leading underscore:
226
- `logger`, SQL constants (`*_SCHEMA`, `SELECT_*`, ...), helpers used only within the module.
227
- A public-looking name is not a claim that the name is exported.
228
- - Tests may import names that are not in `__all__`; white-box testing does not make a name
229
- part of the contract.
230
- - Do not re-export: a name in `__all__` must be defined in that same module.
231
- Import a name from the module that defines it, not from a module that happens to import it.
232
- - `__all__ = ()` is a real claim that nothing outside the module may import from it,
233
- and is correct only for leaf modules.
234
-
235
- Consequence, enforced by `ConventionTests` in `stepup/core/pytest.py`
236
- (subclassed in `tests/test_conventions.py`): any `from .mod import X` inside
237
- `stepup/` requires `X` to be in `mod.__all__`, with no exemption for underscore-prefixed names.
238
- The same holds for a name imported from another package in the `stepup` namespace,
239
- which is how an extension package inherits this convention.
240
- When a module needs something private from another one,
241
- move that name to a module both may depend on (`utils.py` is often the right home),
242
- or promote it to part of the defining module's contract.
243
-
244
264
  ### Data Classes
245
265
 
246
266
  The project uses `attrs` for data classes,
@@ -261,9 +281,15 @@ is already a declared dependency and import it at the top of the file instead.
261
281
  it makes the default log level of `stepup build` `DEBUG` and makes internal consistency checks
262
282
  fatal instead of self-correcting.
263
283
  Profiling output (`STEPUP_BUILD_PERF`, `STEPUP_BUILD_SQLLOG`, `STEPUP_BUILD_JOBLOG`)
264
- can be analyzed with `tools/analyze_perf.py`.
284
+ can be analyzed with `tools/perf/analyze_perf.py`.
285
+
286
+ Three settings select a code path that the default test run does not cover,
287
+ so CI repeats the suite with them (`.github/workflows/pytest.yaml`):
288
+ `STEPUP_RPC_TRANSPORT` (Unix socket or TCP), `STEPUP_WATCHER` and `STEPUP_WATCHER_RECURSIVE`
289
+ (the file watcher library and its mode), and `STEPUP_BUILD_FORKSERVER`.
290
+ `STEPUP_SHELL` names the POSIX shell for shell steps, which matters mostly on Windows.
265
291
 
266
292
  See `docs/reference/configuration.md` for the full list of `STEPUP_*` / `STEPUP_BUILD_*`
267
293
  variables.
268
294
  Most StepUp-3-era `STEPUP_*` variables gained a `STEPUP_BUILD_` prefix in the 4.0 migration;
269
- see `docs/migration/from_3x_to_40.md` before assuming an old name still applies.
295
+ see `docs/migration/from_3x_to_4x.md` before assuming an old name still applies.
@@ -15,8 +15,8 @@ and a minimal example that reproduces the problem.
15
15
 
16
16
  ## Setting up a Development Environment
17
17
 
18
- The [developer notes](https://reproducible-reporting.github.io/stepup-core/development/)
19
- explain how to install the development environment with
18
+ The [developer guide](https://reproducible-reporting.github.io/stepup-core/development/)
19
+ explains how to install the development environment with
20
20
  [uv](https://docs.astral.sh/uv/) and [pre-commit](https://pre-commit.com/),
21
21
  how to run the tests, and how to build the documentation locally.
22
22
 
@@ -13,7 +13,7 @@ exclude .editorconfig
13
13
  exclude .gitignore
14
14
  exclude .markdownlint-cli2.jsonc
15
15
  exclude .pre-commit-config.yaml
16
- exclude mkdocs.yaml
16
+ exclude zensical.toml
17
17
 
18
18
  # Exclude tests for now. (Could be useful later for conda package.)
19
19
  prune tests
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup
3
- Version: 4.0.2
3
+ Version: 4.1.0rc1
4
4
  Summary: A dynamic Python build tool for reproducible workflows
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: LGPL-3.0-or-later
@@ -8,19 +8,21 @@ Project-URL: Documentation, https://reproducible-reporting.github.io/stepup-core
8
8
  Project-URL: Issues, https://github.com/reproducible-reporting/stepup-core/issues
9
9
  Project-URL: Source, https://github.com/reproducible-reporting/stepup-core/
10
10
  Project-URL: Changelog, https://reproducible-reporting.github.io/stepup-core/changelog/
11
- Keywords: asyncio,build automation,build tool,dynamic dependencies,incremental build,parallel,POSIX,Python,reproducibility,reproducible research,StepUp,terminal application,workflow,workflow management
11
+ Keywords: asyncio,build automation,build tool,cross-platform,dynamic dependencies,incremental build,parallel,Python,reproducibility,reproducible research,StepUp,terminal application,workflow,workflow management
12
12
  Classifier: Development Status :: 5 - Production/Stable
13
13
  Classifier: Environment :: Console
14
14
  Classifier: Intended Audience :: Education
15
15
  Classifier: Intended Audience :: Science/Research
16
16
  Classifier: Natural Language :: English
17
- Classifier: Operating System :: POSIX
18
17
  Classifier: Operating System :: MacOS
18
+ Classifier: Operating System :: Microsoft :: Windows
19
+ Classifier: Operating System :: POSIX
19
20
  Classifier: Programming Language :: Python :: 3
20
21
  Classifier: Programming Language :: Python :: 3.11
21
22
  Classifier: Programming Language :: Python :: 3.12
22
23
  Classifier: Programming Language :: Python :: 3.13
23
24
  Classifier: Programming Language :: Python :: 3.14
25
+ Classifier: Programming Language :: Python :: 3.15
24
26
  Classifier: Topic :: Software Development :: Build Tools
25
27
  Requires-Python: >=3.11
26
28
  Description-Content-Type: text/markdown
@@ -33,6 +35,7 @@ Requires-Dist: parse>=1.19.1
33
35
  Requires-Dist: path>=16.14.0
34
36
  Requires-Dist: pyyaml>=6.0
35
37
  Requires-Dist: rich>=13.0.0
38
+ Requires-Dist: watchfiles>=1.3.0; sys_platform != "linux"
36
39
  Provides-Extra: tests
37
40
  Requires-Dist: pytest>=8.2.0; extra == "tests"
38
41
  Requires-Dist: pytest-asyncio>=0.24; extra == "tests"
@@ -41,15 +44,15 @@ Requires-Dist: pytest-xdist>=3.6.1; extra == "tests"
41
44
  Provides-Extra: examples
42
45
  Requires-Dist: matplotlib>=3.8.0; extra == "examples"
43
46
  Requires-Dist: numpy>=1.26.0; extra == "examples"
47
+ Provides-Extra: watchfiles
48
+ Requires-Dist: watchfiles>=1.3.0; extra == "watchfiles"
44
49
  Provides-Extra: dev
45
- Requires-Dist: stepup[examples,tests]; extra == "dev"
46
- Requires-Dist: mkdocs; extra == "dev"
47
- Requires-Dist: mkdocstrings[python]; extra == "dev"
48
- Requires-Dist: mkdocs-macros-plugin; extra == "dev"
49
- Requires-Dist: mkdocs-material; extra == "dev"
50
+ Requires-Dist: stepup[examples,tests,watchfiles]; extra == "dev"
51
+ Requires-Dist: mkdocstrings-python; extra == "dev"
50
52
  Requires-Dist: reuse; extra == "dev"
51
53
  Requires-Dist: snipwise<2.0a0; extra == "dev"
52
54
  Requires-Dist: yappi; extra == "dev"
55
+ Requires-Dist: zensical; extra == "dev"
53
56
  Dynamic: license-file
54
57
 
55
58
  <!--
@@ -61,7 +64,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
61
64
 
62
65
  [![release](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
63
66
  [![pytest](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
64
- [![mkdocs](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
67
+ [![docs](https://github.com/reproducible-reporting/stepup-core/actions/workflows/docs.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/docs.yaml)
65
68
  [![PyPI Version](https://img.shields.io/pypi/v/stepup)](https://pypi.org/project/stepup/)
66
69
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22018869.svg)](https://doi.org/10.5281/zenodo.22018869)
67
70
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup)
@@ -7,7 +7,7 @@ SPDX-License-Identifier: LGPL-3.0-or-later
7
7
 
8
8
  [![release](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
9
9
  [![pytest](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
10
- [![mkdocs](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
10
+ [![docs](https://github.com/reproducible-reporting/stepup-core/actions/workflows/docs.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/docs.yaml)
11
11
  [![PyPI Version](https://img.shields.io/pypi/v/stepup)](https://pypi.org/project/stepup/)
12
12
  [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22018869.svg)](https://doi.org/10.5281/zenodo.22018869)
13
13
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup)
@@ -18,10 +18,10 @@ keywords = [
18
18
  "asyncio",
19
19
  "build automation",
20
20
  "build tool",
21
+ "cross-platform",
21
22
  "dynamic dependencies",
22
23
  "incremental build",
23
24
  "parallel",
24
- "POSIX",
25
25
  "Python",
26
26
  "reproducibility",
27
27
  "reproducible research",
@@ -37,13 +37,15 @@ classifiers = [
37
37
  "Intended Audience :: Education",
38
38
  "Intended Audience :: Science/Research",
39
39
  "Natural Language :: English",
40
- "Operating System :: POSIX",
41
40
  "Operating System :: MacOS",
41
+ "Operating System :: Microsoft :: Windows",
42
+ "Operating System :: POSIX",
42
43
  "Programming Language :: Python :: 3",
43
44
  "Programming Language :: Python :: 3.11",
44
45
  "Programming Language :: Python :: 3.12",
45
46
  "Programming Language :: Python :: 3.13",
46
47
  "Programming Language :: Python :: 3.14",
48
+ "Programming Language :: Python :: 3.15",
47
49
  "Topic :: Software Development :: Build Tools",
48
50
  ]
49
51
  dependencies = [
@@ -55,6 +57,7 @@ dependencies = [
55
57
  "path>=16.14.0",
56
58
  "pyyaml>=6.0",
57
59
  "rich>=13.0.0",
60
+ "watchfiles>=1.3.0 ; sys_platform != 'linux'",
58
61
  ]
59
62
  dynamic = ["version"]
60
63
 
@@ -74,15 +77,18 @@ examples = [
74
77
  "matplotlib>=3.8.0",
75
78
  "numpy>=1.26.0",
76
79
  ]
80
+ # An alternative file watcher on Linux, selected with STEPUP_WATCHER=watchfiles.
81
+ # It is a regular dependency on other platforms.
82
+ watchfiles = [
83
+ "watchfiles>=1.3.0",
84
+ ]
77
85
  dev = [
78
- "stepup[tests,examples]",
79
- "mkdocs",
80
- "mkdocstrings[python]",
81
- "mkdocs-macros-plugin",
82
- "mkdocs-material",
86
+ "stepup[tests,examples,watchfiles]",
87
+ "mkdocstrings-python",
83
88
  "reuse",
84
89
  "snipwise<2.0a0",
85
90
  "yappi",
91
+ "zensical",
86
92
  ]
87
93
 
88
94
  [project.urls]
@@ -97,36 +103,51 @@ sb = "stepup.core.__main__:sb_main"
97
103
  sc-render-jinja = "stepup.core.render_jinja:main"
98
104
 
99
105
  [project.entry-points."stepup.tools"]
100
- boot = "stepup.core.tui:add_boot_subcommand"
101
- browse = "stepup.core.browse:add_browse_subcommand"
102
- build = "stepup.core.tui:add_build_subcommand"
103
- clean = "stepup.core.clean:add_clean_subcommand"
104
- drain = "stepup.core.interact:add_drain_subcommand"
105
- rebuild = "stepup.core.interact:add_rebuild_subcommand"
106
- graph = "stepup.core.interact:add_graph_subcommand"
107
- wait = "stepup.core.interact:add_wait_subcommand"
108
- join = "stepup.core.interact:add_join_subcommand"
109
- config = "stepup.core.config_tool:add_config_subcommand"
110
- shutdown = "stepup.core.interact:add_shutdown_subcommand"
111
- status = "stepup.core.status:add_status_subcommand"
106
+ boot = "stepup.core.cli.build:add_boot_subcommand"
107
+ browse = "stepup.core.cli.browse:add_browse_subcommand"
108
+ build = "stepup.core.cli.build:add_build_subcommand"
109
+ clean = "stepup.core.cli.clean:add_clean_subcommand"
110
+ drain = "stepup.core.cli.interact:add_drain_subcommand"
111
+ rebuild = "stepup.core.cli.interact:add_rebuild_subcommand"
112
+ graph = "stepup.core.cli.interact:add_graph_subcommand"
113
+ wait = "stepup.core.cli.interact:add_wait_subcommand"
114
+ join = "stepup.core.cli.interact:add_join_subcommand"
115
+ config = "stepup.core.cli.config:add_config_subcommand"
116
+ shutdown = "stepup.core.cli.interact:add_shutdown_subcommand"
117
+ status = "stepup.core.cli.status:add_status_subcommand"
112
118
 
113
119
  [tool.pytest.ini_options]
114
120
  addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
115
121
  asyncio_mode = "auto"
116
122
  asyncio_default_fixture_loop_scope = "function"
117
123
  markers = [
118
- "requires_forkserver: mark test that requires STEPUP_BUILD_FORKSERVER != 0",
124
+ "requires_af_unix: skip unless Unix domain sockets (socket.AF_UNIX) are available",
125
+ "requires_fifo: skip unless FIFOs can be created in the file system (os.mkfifo)",
126
+ "requires_file_modes: skip unless files have permission bits that chmod changes",
127
+ "requires_forkserver: skip unless the forkserver is available and STEPUP_BUILD_FORKSERVER != 0",
128
+ "requires_termios: skip unless POSIX terminal control (termios, pty) is available",
129
+ "requires_suspend: skip unless the job-control signals of Ctrl-Z are available",
130
+ "requires_symlinks: skip unless the user may create symbolic links",
131
+ "requires_usr_bin_env: skip unless the file /usr/bin/env exists",
132
+ "requires_windows_console: skip unless the terminal is a Windows console",
119
133
  ]
120
134
  timeout = 15
135
+ # The test modules in subdirectories import helpers from `tests/core_common.py`.
136
+ pythonpath = ["tests"]
121
137
 
122
138
  [tool.ruff]
123
139
  line-length = 100
124
140
  target-version = "py311"
125
141
 
126
142
  [tool.ruff.lint]
143
+ # Preview mode is only enabled for the preview rules selected by their full code below,
144
+ # but it also turns on the preview behavior of stable rules.
145
+ preview = true
146
+ explicit-preview-rules = true
127
147
  select = [
128
148
  "A", "B", "BLE", "C4", "E", "EXE", "F", "I", "ICN", "ISC", "N", "NPY", "PERF", "PIE", "PL",
129
- "PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W"
149
+ "PT", "PYI", "RET", "RSE", "RUF", "SIM", "TRY", "UP", "W",
150
+ "PLW1514", # https://docs.astral.sh/ruff/rules/unspecified-encoding/ (preview in ruff 0.17.2)
130
151
  ]
131
152
  ignore = [
132
153
  "ISC004", # https://docs.astral.sh/ruff/rules/isc004/
@@ -148,8 +169,8 @@ ignore = [
148
169
  "TRY301", # https://docs.astral.sh/ruff/rules/raise-within-try/
149
170
  ]
150
171
 
151
- [tool.setuptools]
152
- packages = ["stepup.core"]
172
+ [tool.setuptools.packages.find]
173
+ include = ["stepup.core", "stepup.core.*"]
153
174
 
154
175
  [tool.setuptools_scm]
155
176
  version_scheme = "post-release"
@@ -163,7 +184,7 @@ mode.strict-config = true
163
184
  extensions.front-matter.enabled = true
164
185
  # Rule md007 (ul-indent) expects a list nested in an ordered list item
165
186
  # to be indented to the parent's content column, i.e. three spaces for a `1. ` item.
166
- # Python-Markdown, and therefore MkDocs, only nests such a list when it is indented four spaces.
187
+ # Python-Markdown, and therefore Zensical, only nests such a list when it is indented four spaces.
167
188
  plugins.md007.enabled = false
168
189
  plugins.md013.line_length = 100
169
190
  plugins.md013.heading_line_length = 100
@@ -22,9 +22,9 @@ render = "{{ content | unwrap }}"
22
22
 
23
23
  # The meta description that every documentation page carries.
24
24
  [[targets]]
25
- patterns = ["mkdocs.yaml"]
25
+ patterns = ["zensical.toml"]
26
26
  scanner = "regex"
27
- regex = '(?m)^site_description: (?P<content>.*)$'
27
+ regex = '(?m)^site_description = "(?P<content>[^"]*)"$'
28
28
  snippets = ["tagline"]
29
29
  render = "{{ content | unwrap }}"
30
30
 
@@ -108,10 +108,10 @@ where the full set of dependencies cannot be determined in advance.
108
108
  asyncio
109
109
  build automation
110
110
  build tool
111
+ cross-platform
111
112
  dynamic dependencies
112
113
  incremental build
113
114
  parallel
114
- POSIX
115
115
  Python
116
116
  reproducibility
117
117
  reproducible research