stepup 3.2.3__tar.gz → 4.0.0rc2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. stepup-4.0.0rc2/CLAUDE.md +333 -0
  2. {stepup-3.2.3/stepup.egg-info → stepup-4.0.0rc2}/PKG-INFO +46 -8
  3. stepup-4.0.0rc2/README.md +54 -0
  4. {stepup-3.2.3 → stepup-4.0.0rc2}/pyproject.toml +12 -10
  5. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/__main__.py +23 -7
  6. stepup-4.0.0rc2/stepup/core/api.py +1395 -0
  7. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/asyncio.py +23 -1
  8. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/browse.py +173 -71
  9. stepup-4.0.0rc2/stepup/core/builder.py +480 -0
  10. stepup-4.0.0rc2/stepup/core/call.py +171 -0
  11. stepup-4.0.0rc2/stepup/core/cattrs.py +43 -0
  12. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/clean.py +92 -46
  13. stepup-4.0.0rc2/stepup/core/config.py +520 -0
  14. stepup-4.0.0rc2/stepup/core/constants.py +50 -0
  15. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/director.py +260 -153
  16. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/enums.py +43 -20
  17. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/exceptions.py +1 -1
  18. stepup-4.0.0rc2/stepup/core/executor.py +1048 -0
  19. stepup-4.0.0rc2/stepup/core/extapi.py +356 -0
  20. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/file.py +69 -85
  21. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/hash.py +71 -56
  22. stepup-4.0.0rc2/stepup/core/hasher.py +205 -0
  23. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/interact.py +40 -30
  24. stepup-4.0.0rc2/stepup/core/job.py +105 -0
  25. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/nglob.py +4 -3
  26. stepup-4.0.0rc2/stepup/core/path.py +238 -0
  27. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/pytest.py +17 -4
  28. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/render_jinja.py +13 -28
  29. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/reporter.py +31 -30
  30. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/rpc.py +35 -21
  31. stepup-4.0.0rc2/stepup/core/scheduler.py +605 -0
  32. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/script.py +15 -11
  33. stepup-4.0.0rc2/stepup/core/sqlite3.py +365 -0
  34. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/startup.py +60 -62
  35. stepup-3.2.3/stepup/core/deferred_glob.py → stepup-4.0.0rc2/stepup/core/static_tree.py +13 -18
  36. stepup-4.0.0rc2/stepup/core/step.py +985 -0
  37. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/stepinfo.py +5 -3
  38. stepup-3.2.3/stepup/core/cascade.py → stepup-4.0.0rc2/stepup/core/trellis.py +242 -321
  39. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/tui.py +184 -65
  40. stepup-4.0.0rc2/stepup/core/utils.py +190 -0
  41. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/watcher.py +143 -81
  42. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/workflow.py +327 -430
  43. {stepup-3.2.3 → stepup-4.0.0rc2/stepup.egg-info}/PKG-INFO +46 -8
  44. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup.egg-info/SOURCES.txt +13 -5
  45. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup.egg-info/entry_points.txt +5 -9
  46. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup.egg-info/requires.txt +2 -1
  47. stepup-4.0.0rc2/stepup.egg-info/scm_file_list.json +2188 -0
  48. stepup-4.0.0rc2/stepup.egg-info/scm_version.json +8 -0
  49. stepup-3.2.3/README.md +0 -17
  50. stepup-3.2.3/stepup/core/actions.py +0 -162
  51. stepup-3.2.3/stepup/core/api.py +0 -1332
  52. stepup-3.2.3/stepup/core/call.py +0 -142
  53. stepup-3.2.3/stepup/core/job.py +0 -183
  54. stepup-3.2.3/stepup/core/runner.py +0 -390
  55. stepup-3.2.3/stepup/core/scheduler.py +0 -194
  56. stepup-3.2.3/stepup/core/sqlite3.py +0 -83
  57. stepup-3.2.3/stepup/core/step.py +0 -992
  58. stepup-3.2.3/stepup/core/utils.py +0 -430
  59. stepup-3.2.3/stepup/core/worker.py +0 -1205
  60. {stepup-3.2.3 → stepup-4.0.0rc2}/LICENSE +0 -0
  61. {stepup-3.2.3 → stepup-4.0.0rc2}/MANIFEST.in +0 -0
  62. {stepup-3.2.3 → stepup-4.0.0rc2}/setup.cfg +0 -0
  63. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/__init__.py +0 -0
  64. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup/core/logo.svg +0 -0
  65. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup.egg-info/dependency_links.txt +0 -0
  66. {stepup-3.2.3 → stepup-4.0.0rc2}/stepup.egg-info/top_level.txt +0 -0
@@ -0,0 +1,333 @@
1
+ # CLAUDE.md
2
+
3
+ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
+
5
+ ## Overview
6
+
7
+ StepUp Core is a dynamic build tool implemented in Python.
8
+ It runs a persistent **director** process that manages a workflow graph (stored in SQLite),
9
+ dispatches jobs to a **step executor**,
10
+ and reacts to file system changes (via inotify) to re-run only outdated steps.
11
+
12
+ ## Commands
13
+
14
+ ### Setup
15
+
16
+ ```bash
17
+ python -m venv venv
18
+ source venv/bin/activate # or: direnv allow (uses .envrc)
19
+ pip install -e .[dev]
20
+ pre-commit install
21
+ ```
22
+
23
+ The `.envrc` sets `STEPUP_DEBUG=1`, `STEPUP_DURATION=0`,
24
+ and `STEPUP_SYNC_RPC_TIMEOUT=30` for development.
25
+
26
+ ### Linting
27
+
28
+ Pre-commit hooks run `ruff format` and `ruff check` automatically on commit.
29
+ After making code changes, run all pre-commit checks before considering the work done:
30
+
31
+ ```bash
32
+ pre-commit run --all
33
+ ```
34
+
35
+ To run individual linters manually:
36
+
37
+ ```bash
38
+ ruff format stepup/ tests/
39
+ ruff check --fix stepup/ tests/
40
+ ```
41
+
42
+ ### Tests
43
+
44
+ ```bash
45
+ pytest -vv # all tests (parallel by default via pytest-xdist)
46
+ pytest -vv tests/test_api.py # single file
47
+ pytest -vv -k "test_name" # single test by name
48
+ ```
49
+
50
+ Tests default to `-n auto --dist worksteal` (parallel).
51
+ The test suite uses `pytest-asyncio` for async tests.
52
+
53
+ ### Documentation
54
+
55
+ ```bash
56
+ mkdocs serve # live preview at http://127.0.0.1:8000/
57
+ ```
58
+
59
+ Note that docstrings are written in Markdown, not reStructuredText!
60
+
61
+ #### Documentation examples
62
+
63
+ Each `docs/getting_started/<example>/` directory contains a `main.sh`
64
+ that generates `stdout.txt` (the terminal output shown in the tutorial page).
65
+ To regenerate after changing example scripts, run:
66
+
67
+ ```bash
68
+ cd docs/getting_started/<example>
69
+ bash main.sh
70
+ ```
71
+
72
+ This runs StepUp locally and captures the output via `sed -f ../../clean_stdout.sed`.
73
+ Commit the updated `stdout.txt` alongside any source changes.
74
+
75
+ ## Coding Conventions
76
+
77
+ ### Linting (ruff)
78
+
79
+ Ruff's rule selection is configured in `pyproject.toml` under `[tool.ruff.lint]`.
80
+ Do not add `# noqa` comments unless the violation is a genuine false positive that cannot
81
+ be resolved by restructuring the code — the project's rule set already excludes rules
82
+ that would fire spuriously in this codebase.
83
+
84
+ Key rules to be aware of:
85
+
86
+ - The default line length is 100.
87
+
88
+ ### Docstrings
89
+
90
+ Use **NumPy-style** sections (`Parameters`, `Returns`, `Raises`, ...)
91
+ Some conventions specific to this codebase:
92
+
93
+ - Docstrings are written in Markdown, not reStructuredText! Some important gotcha's:
94
+ - Use `**bold**` for emphasis, not `*italics*` (which is reserved for parameter names).
95
+ - Use single backticks for inline code and parameter names, not double backticks.
96
+ - Use triple backticks for code blocks,
97
+ and specify the language for syntax highlighting (e.g., ```python).
98
+ - Wrap lines using semantic breaks (e.g., after sentences or logical units),
99
+ not hard-wrapping at a specific character limit.
100
+ See <https://sembr.org/>
101
+ - Use the imperative mood for function descriptions
102
+ (e.g., "Compute the hash of a file."),
103
+ except for `@property` getters where the description should be a noun phrase
104
+ (e.g., "The parent directory path.").
105
+ - Do not repeat type annotations in the docstring — they are already in the function signature.
106
+ - In `Parameters` sections, use the **parameter name** as the heading for each parameter,
107
+ not the type. Grouping closely related parameters under a combined heading
108
+ (e.g., `stdout, stderr`) is allowed when the mkdocs rendering supports it and
109
+ the parameters share the same description.
110
+
111
+ ```python
112
+ In `Returns` sections, use a **semantic name** for the return value, not the type:
113
+
114
+ ```python
115
+ # correct
116
+ Returns
117
+ -------
118
+ parent
119
+ The parent directory path.
120
+
121
+ # wrong — the type is already in the signature
122
+ Returns
123
+ -------
124
+ Path
125
+ The parent directory path.
126
+ ```
127
+
128
+ ### Markdown
129
+
130
+ The project uses markdownlint (via pre-commit) on all `.md` files.
131
+ Two rules that are easy to get wrong:
132
+
133
+ - **MD007** — nested list items must use **4-space** indentation, not 2-space.
134
+ Match the pattern already in use throughout the repo:
135
+
136
+ ```markdown
137
+ - Top-level item
138
+ - Nested item (4 spaces)
139
+ ```
140
+
141
+ - **MD031** — fenced code blocks must have a **blank line** before and after them,
142
+ even when they appear inside a list item.
143
+
144
+ ### Dependencies
145
+
146
+ Runtime dependencies are declared in `pyproject.toml` under `[project] dependencies`.
147
+ Before adding a lazy import or a try/except ImportError guard, check whether the package
148
+ is already a declared dependency and import it at the top of the file instead.
149
+
150
+ ## Architecture
151
+
152
+ ### Process Model
153
+
154
+ StepUp runs as two process types:
155
+
156
+ - **Director** (`director.py`):
157
+ An asyncio process that owns the workflow graph and SQLite database.
158
+ It exposes an RPC server over a Unix socket (`.stepup/sockets/director`).
159
+ Manages `Builder`, `Watcher`, and `Scheduler`.
160
+ Steps run *inside* the director's event loop as asyncio tasks.
161
+ - **Executor** (`executor.py`):
162
+ Runs each step as an asyncio task. Commands run as asyncio subprocesses (shell / direct
163
+ exec) or in a forkserver child (Python scripts and console-script entry points).
164
+ A single `Executor` instance serves all concurrent steps; `--jobs` is the
165
+ concurrency limit. Step child processes call back into the director over its RPC socket
166
+ (e.g. `amend()`, `step()`).
167
+ - **Hashing** (`hasher.py`):
168
+ File/step hashing — the only blocking work — is offloaded to a separate process: a
169
+ forkserver child when `--fork-runpy` is on, otherwise a `_stepup_hasher` subprocess.
170
+ - **TUI** (`tui.py`):
171
+ Boots the director as a subprocess and connects to it via the reporter RPC socket.
172
+ Renders progress to the terminal.
173
+
174
+ The entry point `stepup build` (in `tui.py`) is what users run.
175
+ It spawns the director and connects to it.
176
+
177
+ ### Workflow Graph (`trellis.py`, `workflow.py`)
178
+
179
+ The core data structure is a combined **provenance** and **dependency** graph stored in SQLite.
180
+ `Trellis` (in `trellis.py`) is the abstract base implementing the graph, leveraging recursive SQL.
181
+ `Workflow` (in `workflow.py`) extends it with concrete node types:
182
+
183
+ - **`File`** (`file.py`):
184
+ Tracks files with states `STATIC | AWAITED | BUILT | OUTDATED | MISSING | VOLATILE`.
185
+ - **`Step`** (`step.py`):
186
+ A build step (command + inputs/outputs). States: `PENDING | RUNNING | SUCCEEDED | FAILED`.
187
+ - **`StaticTree`** (`static_tree.py`):
188
+ Static tree node, used for inputs that are automatically declared as static (e.g., source files).
189
+
190
+ All graph mutations happen inside SQLite transactions.
191
+ The `DBSession` in `utils.py` serializes writes.
192
+
193
+ #### Database schema versioning
194
+
195
+ The schema version is `Trellis.schema_version` (in `trellis.py`), written to the database via
196
+ `PRAGMA user_version`. On a version mismatch, the database is **wiped and recreated** from
197
+ scratch (`wipe_database`) — there is no `ALTER TABLE` migration path.
198
+
199
+ **Policy: bump `schema_version` at most once per release.**
200
+ During a pre-release refactor, many commits may change the schema,
201
+ but they all share the single bumped version for the upcoming release;
202
+ do not bump the version again within the same release cycle.
203
+ Record each individual schema change as a comment line in the `schema_version` docstring,
204
+ even when the number itself does not change.
205
+
206
+ ### RPC Layer (`rpc.py`)
207
+
208
+ Lightweight pickle-based RPC over asyncio streams or Unix sockets.
209
+ Methods decorated with `@allow_rpc` are exposed remotely.
210
+ Both sync (`SocketSyncRPCClient`) and async (`AsyncRPCClient`) clients exist.
211
+ The director runs a socket RPC server; step child processes and the TUI are the clients.
212
+
213
+ ### File path considerations
214
+
215
+ StepUp uses the `path` module instead of the built-in `pathlib` to handle file paths.
216
+ In some cases, path affixes must be preserved (leading `./` or trailing `/`),
217
+ which `pathlib` normalizes away. The `path` module preserves these affixes.
218
+
219
+ The affixes are currently used in the places in StepUp:
220
+
221
+ - The `dst` argument of the `copy()` function in `stepup.core.api`, with
222
+ a reusable mechanism for output path construction in `make_path_out()` in `stepup.core.utils`.
223
+ - A local executable must contain at least one slash, e.g., `./script.sh` or `bin/script.sh`.
224
+ - The `getenv()` function in `stepup.core.utils` preserves path affixes
225
+ when reading environment variables must be treated as paths.
226
+ - A static tree path in the database is always stored with a trailing slash.
227
+
228
+ The `get_affixes()` and `apply_affixes()` functions in `stepup.core.utils` are used to
229
+ extract and re-apply the affixes when needed.
230
+
231
+ ### User-Facing API (`api.py`)
232
+
233
+ `plan.py` scripts call functions in `api.py` (e.g., `static()`, `step()`, `glob()`)
234
+ which send RPC calls to the director.
235
+ The module must not be imported by other `stepup.core` modules
236
+ except `interact.py`, `call.py`, `script.py`, and `extapi.py` (local imports only).
237
+
238
+ ### Extension Developer API (`extapi.py`)
239
+
240
+ `extapi.py` collects utilities for authors of StepUp extension packages:
241
+ `subs_env_vars`, `get_rpc_client`, `filter_dependencies`, and `get_local_import_paths`.
242
+ These are re-exported from `api.py` and `utils.py` for backward compatibility.
243
+ `extapi.py` imports from `api.py` only via local (inside-function) imports to avoid
244
+ circular dependencies at module load time.
245
+
246
+ ### Step Execution Pipeline
247
+
248
+ 1. `Scheduler` (`scheduler.py`) picks the highest-priority runnable step
249
+ from the `Workflow` and creates a corresponding `Job` instance.
250
+ 2. `Builder` (`builder.py`) requests a runnable job from the scheduler and, up to the
251
+ concurrency limit, starts it as an asyncio task on the shared `Executor`.
252
+ 3. `Executor` (`executor.py`) runs the step's command (subprocess or forkserver child),
253
+ which may produce more RPC calls back to the director.
254
+ 4. The executor computes file hashes in a separate process and updates
255
+ `FileState` and `StepState` in the workflow.
256
+
257
+ ### Named Globs (`nglob.py`)
258
+
259
+ `NGlobSingle` / `NGlobMulti` implement pattern matching with named back-references (`${*name}`).
260
+ Used in the API for dynamic file discovery with consistency constraints across patterns.
261
+
262
+ ### Key Environment Variables
263
+
264
+ | Variable | Purpose |
265
+ | --- | --- |
266
+ | `STEPUP_DEBUG` | Enable debug logging |
267
+ | `STEPUP_DURATION` | Measure step durations to optimize scheduling (set `0` to disable in tests) |
268
+ | `STEPUP_SYNC_RPC_TIMEOUT` | Timeout for sync RPC calls (seconds) |
269
+ | `STEPUP_PERF` | Frequency (Hz) for Linux `perf` profiling of director |
270
+ | `STEPUP_YAPPI` | Enable Yappi profiling of director |
271
+
272
+ ### Test Structure
273
+
274
+ - `tests/conftest.py` defines fixtures:
275
+ - `wfs` (bare workflow)
276
+ - `wfp` (workflow with plan.py),
277
+ - `client` (full director running in-process).
278
+ - `tests/examples/*/` contains integration test cases,
279
+ each with `plan.py`, `main.sh`, and `expected_stdout*.txt` / `expected_graph*.txt`.
280
+ These are run by `tests/test_examples.py`.
281
+ See `tests/examples/README.md` for a detailed explanation of the `main.sh` conventions
282
+ and how the test builder compares `current_*` files against `expected_*` files.
283
+ - New examples are **not** auto-discovered: register each in the `@pytest.mark.parametrize`
284
+ `name` list of `test_example` in `tests/test_examples.py` (and `test_plan` if the plan
285
+ should run standalone), or it is silently never run.
286
+ - CI runs the example suite twice, with `STEPUP_BUILD_FORK_RUNPY=1` and `=0`, so examples
287
+ must pass under both the forkserver and plain-subprocess paths.
288
+ - The "Standard error" page is replaced with `(stripped)` before comparison, so assert
289
+ stderr text by grepping `.stepup/success.log` (full output) instead of `expected_stdout.txt`.
290
+ - `stepup/core/pytest.py`:
291
+ Pytest helpers for integration tests that run actual StepUp workflows.
292
+ - To regenerate `expected_*` files after an intentional behavior change, run:
293
+
294
+ ```bash
295
+ STEPUP_OVERWRITE_EXPECTED=1 pytest tests/test_examples.py
296
+ ```
297
+
298
+ Review the diffs with `git diff` afterwards to confirm only expected changes.
299
+
300
+ ### Test instructions
301
+
302
+ The following test commands will complete quickly as it skips the integration tests:
303
+
304
+ ```bash
305
+ pytest -k "not test_example"
306
+ ```
307
+
308
+ Always wrap the quick test run in a short timeout, e.g.:
309
+
310
+ ```bash
311
+ timeout 15 pytest -k "not test_example"
312
+ ```
313
+
314
+ 15 seconds is very generous for this selection.
315
+ If a step crashes, the `client` test fixture can otherwise block indefinitely
316
+ (it waits for the workflow to reach a state that never arrives),
317
+ so a timeout prevents a runaway, hanging test process.
318
+
319
+ It may also be useful to run a small number of integration tests,
320
+ to get a first quick feedback on the overall system:
321
+
322
+ ```bash
323
+ pytest tests/test_examples.py -k "test_example[no_static] or test_example[restart_add_missing]"
324
+ ```
325
+
326
+ These are two simple examples that run quickly and will fail when the core system is broken.
327
+ A full run with all integration tests takes several minutes and is best run as a final check only.
328
+
329
+ ### Release Process
330
+
331
+ 1. Update `docs/changelog.md`.
332
+ 2. Commit and tag: `git tag vX.Y.Z`.
333
+ 3. Push with tags: `git push origin main --tags` — triggers PyPI GitHub Action.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: stepup
3
- Version: 3.2.3
3
+ Version: 4.0.0rc2
4
4
  Summary: StepUp Core provides the basic framework for the StepUp build tool
5
5
  Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
6
6
  License-Expression: GPL-3.0-or-later
@@ -25,7 +25,8 @@ Requires-Python: >=3.11
25
25
  Description-Content-Type: text/markdown
26
26
  License-File: LICENSE
27
27
  Requires-Dist: asyncinotify>=4.0.9
28
- Requires-Dist: attrs>=22.1.0
28
+ Requires-Dist: attrs>=23.1.0
29
+ Requires-Dist: cattrs>=23.2.3
29
30
  Requires-Dist: jinja2>=3.1.6
30
31
  Requires-Dist: parse>=1.19.1
31
32
  Requires-Dist: path>=16.14.0
@@ -49,16 +50,53 @@ Dynamic: license-file
49
50
 
50
51
  # StepUp Core
51
52
 
52
- [![release](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
53
- [![pytest](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
54
- [![mkdocs](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml/badge.svg)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
53
+ [![release](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
54
+ [![pytest](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
55
+ [![mkdocs](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
55
56
  [![PyPI Version](https://img.shields.io/pypi/v/stepup)](https://pypi.org/project/stepup/)
56
57
  ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup)
57
58
  ![GPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-core)
58
59
  [![CodeFactor](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core/badge)](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
59
60
 
60
- StepUp is a simple, powerful and universal build tool.
61
+ StepUp is a simple, powerful and universal build tool,
62
+ a modern alternative to [Make](https://en.wikipedia.org/wiki/Make_(software)).
63
+ Its defining feature is that it treats the generation and execution of the build workflow
64
+ as one and the same thing:
65
+ a `plan.py` Python script issues build steps via RPC to a persistent background process,
66
+ so steps can be defined on the fly using the outputs of earlier steps.
67
+ This makes StepUp a good fit for builds where the full set of dependencies
68
+ cannot be known in advance.
69
+
61
70
  StepUp Core provides the basic framework for StepUp, without any domain-specific features.
71
+ Those live in extension packages:
72
+
73
+ - [StepUp RepRep](https://reproducible-reporting.github.io/stepup-reprep/)
74
+ for creating **rep**roducible **rep**orts: papers, presentations, theses, etc.
75
+ - [StepUp Queue](https://reproducible-reporting.github.io/stepup-queue/)
76
+ submits jobs to a SLURM scheduler.
77
+
78
+ ## Installation
79
+
80
+ ```bash
81
+ pip install stepup
82
+ ```
83
+
84
+ See the [installation guide](https://reproducible-reporting.github.io/stepup-core/installation/)
85
+ for details.
86
+
87
+ ## Quick Visual Impression
88
+
89
+ [![asciicast](https://asciinema.org/a/718833.svg)](https://asciinema.org/a/718833)
90
+
91
+ ## Documentation
92
+
93
+ Full documentation, including a tutorial and a feature overview,
94
+ is available at <https://reproducible-reporting.github.io/stepup-core>.
95
+
96
+ ## License
62
97
 
63
- For more information, consult the
64
- [documentation](http://reproducible-reporting.github.io/stepup-core).
98
+ StepUp Core is distributed under the terms of the
99
+ [GPL-3.0-or-later](LICENSE) license.
100
+ Contributions are welcome;
101
+ see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
102
+ to get started.
@@ -0,0 +1,54 @@
1
+ <!-- markdownlint-disable line-length -->
2
+
3
+ # StepUp Core
4
+
5
+ [![release](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
6
+ [![pytest](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
7
+ [![mkdocs](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml/badge.svg?branch=main)](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
8
+ [![PyPI Version](https://img.shields.io/pypi/v/stepup)](https://pypi.org/project/stepup/)
9
+ ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/stepup)
10
+ ![GPL-3 License](https://img.shields.io/github/license/reproducible-reporting/stepup-core)
11
+ [![CodeFactor](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core/badge)](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
12
+
13
+ StepUp is a simple, powerful and universal build tool,
14
+ a modern alternative to [Make](https://en.wikipedia.org/wiki/Make_(software)).
15
+ Its defining feature is that it treats the generation and execution of the build workflow
16
+ as one and the same thing:
17
+ a `plan.py` Python script issues build steps via RPC to a persistent background process,
18
+ so steps can be defined on the fly using the outputs of earlier steps.
19
+ This makes StepUp a good fit for builds where the full set of dependencies
20
+ cannot be known in advance.
21
+
22
+ StepUp Core provides the basic framework for StepUp, without any domain-specific features.
23
+ Those live in extension packages:
24
+
25
+ - [StepUp RepRep](https://reproducible-reporting.github.io/stepup-reprep/)
26
+ for creating **rep**roducible **rep**orts: papers, presentations, theses, etc.
27
+ - [StepUp Queue](https://reproducible-reporting.github.io/stepup-queue/)
28
+ submits jobs to a SLURM scheduler.
29
+
30
+ ## Installation
31
+
32
+ ```bash
33
+ pip install stepup
34
+ ```
35
+
36
+ See the [installation guide](https://reproducible-reporting.github.io/stepup-core/installation/)
37
+ for details.
38
+
39
+ ## Quick Visual Impression
40
+
41
+ [![asciicast](https://asciinema.org/a/718833.svg)](https://asciinema.org/a/718833)
42
+
43
+ ## Documentation
44
+
45
+ Full documentation, including a tutorial and a feature overview,
46
+ is available at <https://reproducible-reporting.github.io/stepup-core>.
47
+
48
+ ## License
49
+
50
+ StepUp Core is distributed under the terms of the
51
+ [GPL-3.0-or-later](LICENSE) license.
52
+ Contributions are welcome;
53
+ see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
54
+ to get started.
@@ -29,7 +29,8 @@ classifiers = [
29
29
  dependencies = [
30
30
  # Ensure changes to these dependencies are reflected in .github/requirements-old.txt
31
31
  "asyncinotify>=4.0.9",
32
- "attrs>=22.1.0",
32
+ "attrs>=23.1.0",
33
+ "cattrs>=23.2.3",
33
34
  "jinja2>=3.1.6",
34
35
  "parse>=1.19.1",
35
36
  "path>=16.14.0",
@@ -61,11 +62,14 @@ Changelog = "https://reproducible-reporting.github.io/stepup-core/changelog/"
61
62
 
62
63
  [project.scripts]
63
64
  stepup = "stepup.core.__main__:main"
65
+ sb = "stepup.core.__main__:sb_main"
66
+ _stepup_hasher = "stepup.core.hasher:hasher_tool"
67
+ render-jinja = "stepup.core.render_jinja:main"
64
68
 
65
69
  [project.entry-points."stepup.tools"]
66
- act = "stepup.core.actions:act_subcommand"
67
- browse = "stepup.core.browse:browse_subcommand"
68
70
  boot = "stepup.core.tui:boot_subcommand"
71
+ browse = "stepup.core.browse:browse_subcommand"
72
+ build = "stepup.core.tui:build_subcommand"
69
73
  clean = "stepup.core.clean:clean_subcommand"
70
74
  drain = "stepup.core.interact:drain_subcommand"
71
75
  run = "stepup.core.interact:run_subcommand"
@@ -74,20 +78,18 @@ watch-update = "stepup.core.interact:watch_update_subcommand"
74
78
  watch-delete = "stepup.core.interact:watch_delete_subcommand"
75
79
  wait = "stepup.core.interact:wait_subcommand"
76
80
  join = "stepup.core.interact:join_subcommand"
77
- render-jinja = "stepup.core.render_jinja:render_jinja_subcommand"
81
+ show-config = "stepup.core.config:show_config_subcommand"
78
82
  shutdown = "stepup.core.interact:shutdown_subcommand"
79
83
  status = "stepup.core.interact:status_subcommand"
80
84
 
81
- [project.entry-points."stepup.actions"]
82
- copy = "stepup.core.actions:copy"
83
- mkdir = "stepup.core.actions:mkdir"
84
- render-jinja = "stepup.core.render_jinja:render_jinja_action"
85
- runpy = "stepup.core.actions:runpy"
86
- runsh = "stepup.core.actions:runsh"
87
85
 
88
86
  [tool.pytest.ini_options]
89
87
  addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
88
+ asyncio_mode = "auto"
90
89
  asyncio_default_fixture_loop_scope = "function"
90
+ markers = [
91
+ "requires_fork_runpy: mark test that requires STEPUP_BUILD_FORK_RUNPY != 0",
92
+ ]
91
93
 
92
94
  [tool.ruff]
93
95
  line-length = 100
@@ -27,6 +27,7 @@ from importlib.metadata import version as get_version
27
27
 
28
28
  from path import Path
29
29
 
30
+ from .config import ConfigLoader
30
31
  from .utils import string_to_bool
31
32
 
32
33
 
@@ -40,7 +41,27 @@ def main():
40
41
  parser.print_help()
41
42
 
42
43
 
44
+ def sb_main():
45
+ """Shortcut for `stepup build` (accepts the same arguments)."""
46
+ sys.argv.insert(1, "build")
47
+ main()
48
+
49
+
43
50
  def build_parser() -> tuple[argparse.ArgumentParser, list[callable]]:
51
+ # Configuration loader
52
+ stepup_root = Path(os.getenv("STEPUP_ROOT", os.getcwd()))
53
+ loader = ConfigLoader(
54
+ prefix="stepup",
55
+ config_paths=[
56
+ "/etc/stepup.toml",
57
+ "~/.config/stepup.toml",
58
+ stepup_root / ".stepup.toml",
59
+ stepup_root / "stepup.toml",
60
+ stepup_root / "pyproject.toml",
61
+ stepup_root / "stepup-local.toml",
62
+ ],
63
+ )
64
+
44
65
  # Base argument parser
45
66
  parser = argparse.ArgumentParser(
46
67
  prog="stepup",
@@ -56,12 +77,7 @@ def build_parser() -> tuple[argparse.ArgumentParser, list[callable]]:
56
77
  choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
57
78
  help="Set the logging level. [default=%(default)s]",
58
79
  )
59
- parser.add_argument(
60
- "--root",
61
- type=Path,
62
- default=Path(os.getenv("STEPUP_ROOT", os.getcwd())),
63
- help="Directory containing top-level plan.py [default=%(default)s]",
64
- )
80
+ loader.patch_parser(parser, use_section=False)
65
81
 
66
82
  # Load tool entry points
67
83
  subparsers = parser.add_subparsers(dest="tool", required=False)
@@ -69,7 +85,7 @@ def build_parser() -> tuple[argparse.ArgumentParser, list[callable]]:
69
85
  tool_funcs = {}
70
86
  for tool_ep in tool_eps:
71
87
  tool = tool_ep.load()
72
- tool_funcs[tool_ep.name] = tool(subparsers)
88
+ tool_funcs[tool_ep.name] = tool(subparsers, loader)
73
89
 
74
90
  return parser, tool_funcs
75
91