stepup 3.2.2__tar.gz → 4.0.0rc1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stepup-4.0.0rc1/CLAUDE.md +333 -0
- {stepup-3.2.2/stepup.egg-info → stepup-4.0.0rc1}/PKG-INFO +46 -8
- stepup-4.0.0rc1/README.md +54 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/pyproject.toml +12 -10
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/__main__.py +23 -7
- stepup-4.0.0rc1/stepup/core/api.py +1395 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/asyncio.py +23 -1
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/browse.py +175 -71
- stepup-4.0.0rc1/stepup/core/builder.py +480 -0
- stepup-4.0.0rc1/stepup/core/call.py +171 -0
- stepup-4.0.0rc1/stepup/core/cattrs.py +43 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/clean.py +92 -45
- stepup-4.0.0rc1/stepup/core/config.py +519 -0
- stepup-4.0.0rc1/stepup/core/constants.py +50 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/director.py +260 -153
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/enums.py +43 -20
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/exceptions.py +1 -1
- stepup-4.0.0rc1/stepup/core/executor.py +1048 -0
- stepup-4.0.0rc1/stepup/core/extapi.py +342 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/file.py +69 -81
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/hash.py +71 -56
- stepup-4.0.0rc1/stepup/core/hasher.py +205 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/interact.py +40 -30
- stepup-4.0.0rc1/stepup/core/job.py +105 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/nglob.py +4 -3
- stepup-4.0.0rc1/stepup/core/path.py +234 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/pytest.py +17 -4
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/render_jinja.py +13 -28
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/reporter.py +31 -30
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/rpc.py +35 -21
- stepup-4.0.0rc1/stepup/core/scheduler.py +605 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/script.py +15 -11
- stepup-4.0.0rc1/stepup/core/sqlite3.py +365 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/startup.py +59 -61
- stepup-3.2.2/stepup/core/deferred_glob.py → stepup-4.0.0rc1/stepup/core/static_tree.py +13 -18
- stepup-4.0.0rc1/stepup/core/step.py +985 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/stepinfo.py +5 -3
- stepup-3.2.2/stepup/core/cascade.py → stepup-4.0.0rc1/stepup/core/trellis.py +242 -318
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/tui.py +184 -65
- stepup-4.0.0rc1/stepup/core/utils.py +190 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/watcher.py +143 -81
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/workflow.py +329 -430
- {stepup-3.2.2 → stepup-4.0.0rc1/stepup.egg-info}/PKG-INFO +46 -8
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup.egg-info/SOURCES.txt +14 -5
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup.egg-info/entry_points.txt +5 -9
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup.egg-info/requires.txt +2 -1
- stepup-4.0.0rc1/stepup.egg-info/scm_file_list.json +2188 -0
- stepup-4.0.0rc1/stepup.egg-info/scm_version.json +8 -0
- stepup-3.2.2/README.md +0 -17
- stepup-3.2.2/stepup/core/actions.py +0 -162
- stepup-3.2.2/stepup/core/api.py +0 -1332
- stepup-3.2.2/stepup/core/call.py +0 -142
- stepup-3.2.2/stepup/core/job.py +0 -183
- stepup-3.2.2/stepup/core/runner.py +0 -390
- stepup-3.2.2/stepup/core/scheduler.py +0 -194
- stepup-3.2.2/stepup/core/step.py +0 -992
- stepup-3.2.2/stepup/core/utils.py +0 -446
- stepup-3.2.2/stepup/core/worker.py +0 -1205
- {stepup-3.2.2 → stepup-4.0.0rc1}/LICENSE +0 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/MANIFEST.in +0 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/setup.cfg +0 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/__init__.py +0 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup/core/logo.svg +0 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup.egg-info/dependency_links.txt +0 -0
- {stepup-3.2.2 → stepup-4.0.0rc1}/stepup.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Overview
|
|
6
|
+
|
|
7
|
+
StepUp Core is a dynamic build tool implemented in Python.
|
|
8
|
+
It runs a persistent **director** process that manages a workflow graph (stored in SQLite),
|
|
9
|
+
dispatches jobs to a **step executor**,
|
|
10
|
+
and reacts to file system changes (via inotify) to re-run only outdated steps.
|
|
11
|
+
|
|
12
|
+
## Commands
|
|
13
|
+
|
|
14
|
+
### Setup
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
python -m venv venv
|
|
18
|
+
source venv/bin/activate # or: direnv allow (uses .envrc)
|
|
19
|
+
pip install -e .[dev]
|
|
20
|
+
pre-commit install
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
The `.envrc` sets `STEPUP_DEBUG=1`, `STEPUP_DURATION=0`,
|
|
24
|
+
and `STEPUP_SYNC_RPC_TIMEOUT=30` for development.
|
|
25
|
+
|
|
26
|
+
### Linting
|
|
27
|
+
|
|
28
|
+
Pre-commit hooks run `ruff format` and `ruff check` automatically on commit.
|
|
29
|
+
After making code changes, run all pre-commit checks before considering the work done:
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
pre-commit run --all
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
To run individual linters manually:
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
ruff format stepup/ tests/
|
|
39
|
+
ruff check --fix stepup/ tests/
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
### Tests
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pytest -vv # all tests (parallel by default via pytest-xdist)
|
|
46
|
+
pytest -vv tests/test_api.py # single file
|
|
47
|
+
pytest -vv -k "test_name" # single test by name
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Tests default to `-n auto --dist worksteal` (parallel).
|
|
51
|
+
The test suite uses `pytest-asyncio` for async tests.
|
|
52
|
+
|
|
53
|
+
### Documentation
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
mkdocs serve # live preview at http://127.0.0.1:8000/
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Note that docstrings are written in Markdown, not reStructuredText!
|
|
60
|
+
|
|
61
|
+
#### Documentation examples
|
|
62
|
+
|
|
63
|
+
Each `docs/getting_started/<example>/` directory contains a `main.sh`
|
|
64
|
+
that generates `stdout.txt` (the terminal output shown in the tutorial page).
|
|
65
|
+
To regenerate after changing example scripts, run:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
cd docs/getting_started/<example>
|
|
69
|
+
bash main.sh
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
This runs StepUp locally and captures the output via `sed -f ../../clean_stdout.sed`.
|
|
73
|
+
Commit the updated `stdout.txt` alongside any source changes.
|
|
74
|
+
|
|
75
|
+
## Coding Conventions
|
|
76
|
+
|
|
77
|
+
### Linting (ruff)
|
|
78
|
+
|
|
79
|
+
Ruff's rule selection is configured in `pyproject.toml` under `[tool.ruff.lint]`.
|
|
80
|
+
Do not add `# noqa` comments unless the violation is a genuine false positive that cannot
|
|
81
|
+
be resolved by restructuring the code — the project's rule set already excludes rules
|
|
82
|
+
that would fire spuriously in this codebase.
|
|
83
|
+
|
|
84
|
+
Key rules to be aware of:
|
|
85
|
+
|
|
86
|
+
- The default line length is 100.
|
|
87
|
+
|
|
88
|
+
### Docstrings
|
|
89
|
+
|
|
90
|
+
Use **NumPy-style** sections (`Parameters`, `Returns`, `Raises`, ...)
|
|
91
|
+
Some conventions specific to this codebase:
|
|
92
|
+
|
|
93
|
+
- Docstrings are written in Markdown, not reStructuredText! Some important gotcha's:
|
|
94
|
+
- Use `**bold**` for emphasis, not `*italics*` (which is reserved for parameter names).
|
|
95
|
+
- Use single backticks for inline code and parameter names, not double backticks.
|
|
96
|
+
- Use triple backticks for code blocks,
|
|
97
|
+
and specify the language for syntax highlighting (e.g., ```python).
|
|
98
|
+
- Wrap lines using semantic breaks (e.g., after sentences or logical units),
|
|
99
|
+
not hard-wrapping at a specific character limit.
|
|
100
|
+
See <https://sembr.org/>
|
|
101
|
+
- Use the imperative mood for function descriptions
|
|
102
|
+
(e.g., "Compute the hash of a file."),
|
|
103
|
+
except for `@property` getters where the description should be a noun phrase
|
|
104
|
+
(e.g., "The parent directory path.").
|
|
105
|
+
- Do not repeat type annotations in the docstring — they are already in the function signature.
|
|
106
|
+
- In `Parameters` sections, use the **parameter name** as the heading for each parameter,
|
|
107
|
+
not the type. Grouping closely related parameters under a combined heading
|
|
108
|
+
(e.g., `stdout, stderr`) is allowed when the mkdocs rendering supports it and
|
|
109
|
+
the parameters share the same description.
|
|
110
|
+
|
|
111
|
+
```python
|
|
112
|
+
In `Returns` sections, use a **semantic name** for the return value, not the type:
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
# correct
|
|
116
|
+
Returns
|
|
117
|
+
-------
|
|
118
|
+
parent
|
|
119
|
+
The parent directory path.
|
|
120
|
+
|
|
121
|
+
# wrong — the type is already in the signature
|
|
122
|
+
Returns
|
|
123
|
+
-------
|
|
124
|
+
Path
|
|
125
|
+
The parent directory path.
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### Markdown
|
|
129
|
+
|
|
130
|
+
The project uses markdownlint (via pre-commit) on all `.md` files.
|
|
131
|
+
Two rules that are easy to get wrong:
|
|
132
|
+
|
|
133
|
+
- **MD007** — nested list items must use **4-space** indentation, not 2-space.
|
|
134
|
+
Match the pattern already in use throughout the repo:
|
|
135
|
+
|
|
136
|
+
```markdown
|
|
137
|
+
- Top-level item
|
|
138
|
+
- Nested item (4 spaces)
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
- **MD031** — fenced code blocks must have a **blank line** before and after them,
|
|
142
|
+
even when they appear inside a list item.
|
|
143
|
+
|
|
144
|
+
### Dependencies
|
|
145
|
+
|
|
146
|
+
Runtime dependencies are declared in `pyproject.toml` under `[project] dependencies`.
|
|
147
|
+
Before adding a lazy import or a try/except ImportError guard, check whether the package
|
|
148
|
+
is already a declared dependency and import it at the top of the file instead.
|
|
149
|
+
|
|
150
|
+
## Architecture
|
|
151
|
+
|
|
152
|
+
### Process Model
|
|
153
|
+
|
|
154
|
+
StepUp runs as two process types:
|
|
155
|
+
|
|
156
|
+
- **Director** (`director.py`):
|
|
157
|
+
An asyncio process that owns the workflow graph and SQLite database.
|
|
158
|
+
It exposes an RPC server over a Unix socket (`.stepup/sockets/director`).
|
|
159
|
+
Manages `Builder`, `Watcher`, and `Scheduler`.
|
|
160
|
+
Steps run *inside* the director's event loop as asyncio tasks.
|
|
161
|
+
- **Executor** (`executor.py`):
|
|
162
|
+
Runs each step as an asyncio task. Commands run as asyncio subprocesses (shell / direct
|
|
163
|
+
exec) or in a forkserver child (Python scripts and console-script entry points).
|
|
164
|
+
A single `Executor` instance serves all concurrent steps; `--jobs` is the
|
|
165
|
+
concurrency limit. Step child processes call back into the director over its RPC socket
|
|
166
|
+
(e.g. `amend()`, `step()`).
|
|
167
|
+
- **Hashing** (`hasher.py`):
|
|
168
|
+
File/step hashing — the only blocking work — is offloaded to a separate process: a
|
|
169
|
+
forkserver child when `--fork-runpy` is on, otherwise a `_stepup_hasher` subprocess.
|
|
170
|
+
- **TUI** (`tui.py`):
|
|
171
|
+
Boots the director as a subprocess and connects to it via the reporter RPC socket.
|
|
172
|
+
Renders progress to the terminal.
|
|
173
|
+
|
|
174
|
+
The entry point `stepup build` (in `tui.py`) is what users run.
|
|
175
|
+
It spawns the director and connects to it.
|
|
176
|
+
|
|
177
|
+
### Workflow Graph (`trellis.py`, `workflow.py`)
|
|
178
|
+
|
|
179
|
+
The core data structure is a combined **provenance** and **dependency** graph stored in SQLite.
|
|
180
|
+
`Trellis` (in `trellis.py`) is the abstract base implementing the graph, leveraging recursive SQL.
|
|
181
|
+
`Workflow` (in `workflow.py`) extends it with concrete node types:
|
|
182
|
+
|
|
183
|
+
- **`File`** (`file.py`):
|
|
184
|
+
Tracks files with states `STATIC | AWAITED | BUILT | OUTDATED | MISSING | VOLATILE`.
|
|
185
|
+
- **`Step`** (`step.py`):
|
|
186
|
+
A build step (command + inputs/outputs). States: `PENDING | RUNNING | SUCCEEDED | FAILED`.
|
|
187
|
+
- **`StaticTree`** (`static_tree.py`):
|
|
188
|
+
Static tree node, used for inputs that are automatically declared as static (e.g., source files).
|
|
189
|
+
|
|
190
|
+
All graph mutations happen inside SQLite transactions.
|
|
191
|
+
The `DBSession` in `utils.py` serializes writes.
|
|
192
|
+
|
|
193
|
+
#### Database schema versioning
|
|
194
|
+
|
|
195
|
+
The schema version is `Trellis.schema_version` (in `trellis.py`), written to the database via
|
|
196
|
+
`PRAGMA user_version`. On a version mismatch, the database is **wiped and recreated** from
|
|
197
|
+
scratch (`wipe_database`) — there is no `ALTER TABLE` migration path.
|
|
198
|
+
|
|
199
|
+
**Policy: bump `schema_version` at most once per release.**
|
|
200
|
+
During a pre-release refactor, many commits may change the schema,
|
|
201
|
+
but they all share the single bumped version for the upcoming release;
|
|
202
|
+
do not bump the version again within the same release cycle.
|
|
203
|
+
Record each individual schema change as a comment line in the `schema_version` docstring,
|
|
204
|
+
even when the number itself does not change.
|
|
205
|
+
|
|
206
|
+
### RPC Layer (`rpc.py`)
|
|
207
|
+
|
|
208
|
+
Lightweight pickle-based RPC over asyncio streams or Unix sockets.
|
|
209
|
+
Methods decorated with `@allow_rpc` are exposed remotely.
|
|
210
|
+
Both sync (`SocketSyncRPCClient`) and async (`AsyncRPCClient`) clients exist.
|
|
211
|
+
The director runs a socket RPC server; step child processes and the TUI are the clients.
|
|
212
|
+
|
|
213
|
+
### File path considerations
|
|
214
|
+
|
|
215
|
+
StepUp uses the `path` module instead of the built-in `pathlib` to handle file paths.
|
|
216
|
+
In some cases, path affixes must be preserved (leading `./` or trailing `/`),
|
|
217
|
+
which `pathlib` normalizes away. The `path` module preserves these affixes.
|
|
218
|
+
|
|
219
|
+
The affixes are currently used in the places in StepUp:
|
|
220
|
+
|
|
221
|
+
- The `dst` argument of the `copy()` function in `stepup.core.api`, with
|
|
222
|
+
a reusable mechanism for output path construction in `make_path_out()` in `stepup.core.utils`.
|
|
223
|
+
- A local executable must contain at least one slash, e.g., `./script.sh` or `bin/script.sh`.
|
|
224
|
+
- The `getenv()` function in `stepup.core.utils` preserves path affixes
|
|
225
|
+
when reading environment variables must be treated as paths.
|
|
226
|
+
- A static tree path in the database is always stored with a trailing slash.
|
|
227
|
+
|
|
228
|
+
The `get_affixes()` and `apply_affixes()` functions in `stepup.core.utils` are used to
|
|
229
|
+
extract and re-apply the affixes when needed.
|
|
230
|
+
|
|
231
|
+
### User-Facing API (`api.py`)
|
|
232
|
+
|
|
233
|
+
`plan.py` scripts call functions in `api.py` (e.g., `static()`, `step()`, `glob()`)
|
|
234
|
+
which send RPC calls to the director.
|
|
235
|
+
The module must not be imported by other `stepup.core` modules
|
|
236
|
+
except `interact.py`, `call.py`, `script.py`, and `extapi.py` (local imports only).
|
|
237
|
+
|
|
238
|
+
### Extension Developer API (`extapi.py`)
|
|
239
|
+
|
|
240
|
+
`extapi.py` collects utilities for authors of StepUp extension packages:
|
|
241
|
+
`subs_env_vars`, `get_rpc_client`, `filter_dependencies`, and `get_local_import_paths`.
|
|
242
|
+
These are re-exported from `api.py` and `utils.py` for backward compatibility.
|
|
243
|
+
`extapi.py` imports from `api.py` only via local (inside-function) imports to avoid
|
|
244
|
+
circular dependencies at module load time.
|
|
245
|
+
|
|
246
|
+
### Step Execution Pipeline
|
|
247
|
+
|
|
248
|
+
1. `Scheduler` (`scheduler.py`) picks the highest-priority runnable step
|
|
249
|
+
from the `Workflow` and creates a corresponding `Job` instance.
|
|
250
|
+
2. `Builder` (`builder.py`) requests a runnable job from the scheduler and, up to the
|
|
251
|
+
concurrency limit, starts it as an asyncio task on the shared `Executor`.
|
|
252
|
+
3. `Executor` (`executor.py`) runs the step's command (subprocess or forkserver child),
|
|
253
|
+
which may produce more RPC calls back to the director.
|
|
254
|
+
4. The executor computes file hashes in a separate process and updates
|
|
255
|
+
`FileState` and `StepState` in the workflow.
|
|
256
|
+
|
|
257
|
+
### Named Globs (`nglob.py`)
|
|
258
|
+
|
|
259
|
+
`NGlobSingle` / `NGlobMulti` implement pattern matching with named back-references (`${*name}`).
|
|
260
|
+
Used in the API for dynamic file discovery with consistency constraints across patterns.
|
|
261
|
+
|
|
262
|
+
### Key Environment Variables
|
|
263
|
+
|
|
264
|
+
| Variable | Purpose |
|
|
265
|
+
| --- | --- |
|
|
266
|
+
| `STEPUP_DEBUG` | Enable debug logging |
|
|
267
|
+
| `STEPUP_DURATION` | Measure step durations to optimize scheduling (set `0` to disable in tests) |
|
|
268
|
+
| `STEPUP_SYNC_RPC_TIMEOUT` | Timeout for sync RPC calls (seconds) |
|
|
269
|
+
| `STEPUP_PERF` | Frequency (Hz) for Linux `perf` profiling of director |
|
|
270
|
+
| `STEPUP_YAPPI` | Enable Yappi profiling of director |
|
|
271
|
+
|
|
272
|
+
### Test Structure
|
|
273
|
+
|
|
274
|
+
- `tests/conftest.py` defines fixtures:
|
|
275
|
+
- `wfs` (bare workflow)
|
|
276
|
+
- `wfp` (workflow with plan.py),
|
|
277
|
+
- `client` (full director running in-process).
|
|
278
|
+
- `tests/examples/*/` contains integration test cases,
|
|
279
|
+
each with `plan.py`, `main.sh`, and `expected_stdout*.txt` / `expected_graph*.txt`.
|
|
280
|
+
These are run by `tests/test_examples.py`.
|
|
281
|
+
See `tests/examples/README.md` for a detailed explanation of the `main.sh` conventions
|
|
282
|
+
and how the test builder compares `current_*` files against `expected_*` files.
|
|
283
|
+
- New examples are **not** auto-discovered: register each in the `@pytest.mark.parametrize`
|
|
284
|
+
`name` list of `test_example` in `tests/test_examples.py` (and `test_plan` if the plan
|
|
285
|
+
should run standalone), or it is silently never run.
|
|
286
|
+
- CI runs the example suite twice, with `STEPUP_BUILD_FORK_RUNPY=1` and `=0`, so examples
|
|
287
|
+
must pass under both the forkserver and plain-subprocess paths.
|
|
288
|
+
- The "Standard error" page is replaced with `(stripped)` before comparison, so assert
|
|
289
|
+
stderr text by grepping `.stepup/success.log` (full output) instead of `expected_stdout.txt`.
|
|
290
|
+
- `stepup/core/pytest.py`:
|
|
291
|
+
Pytest helpers for integration tests that run actual StepUp workflows.
|
|
292
|
+
- To regenerate `expected_*` files after an intentional behavior change, run:
|
|
293
|
+
|
|
294
|
+
```bash
|
|
295
|
+
STEPUP_OVERWRITE_EXPECTED=1 pytest tests/test_examples.py
|
|
296
|
+
```
|
|
297
|
+
|
|
298
|
+
Review the diffs with `git diff` afterwards to confirm only expected changes.
|
|
299
|
+
|
|
300
|
+
### Test instructions
|
|
301
|
+
|
|
302
|
+
The following test commands will complete quickly as it skips the integration tests:
|
|
303
|
+
|
|
304
|
+
```bash
|
|
305
|
+
pytest -k "not test_example"
|
|
306
|
+
```
|
|
307
|
+
|
|
308
|
+
Always wrap the quick test run in a short timeout, e.g.:
|
|
309
|
+
|
|
310
|
+
```bash
|
|
311
|
+
timeout 15 pytest -k "not test_example"
|
|
312
|
+
```
|
|
313
|
+
|
|
314
|
+
15 seconds is very generous for this selection.
|
|
315
|
+
If a step crashes, the `client` test fixture can otherwise block indefinitely
|
|
316
|
+
(it waits for the workflow to reach a state that never arrives),
|
|
317
|
+
so a timeout prevents a runaway, hanging test process.
|
|
318
|
+
|
|
319
|
+
It may also be useful to run a small number of integration tests,
|
|
320
|
+
to get a first quick feedback on the overall system:
|
|
321
|
+
|
|
322
|
+
```bash
|
|
323
|
+
pytest tests/test_examples.py -k "test_example[no_static] or test_example[restart_add_missing]"
|
|
324
|
+
```
|
|
325
|
+
|
|
326
|
+
These are two simple examples that run quickly and will fail when the core system is broken.
|
|
327
|
+
A full run with all integration tests takes several minutes and is best run as a final check only.
|
|
328
|
+
|
|
329
|
+
### Release Process
|
|
330
|
+
|
|
331
|
+
1. Update `docs/changelog.md`.
|
|
332
|
+
2. Commit and tag: `git tag vX.Y.Z`.
|
|
333
|
+
3. Push with tags: `git push origin main --tags` — triggers PyPI GitHub Action.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: stepup
|
|
3
|
-
Version:
|
|
3
|
+
Version: 4.0.0rc1
|
|
4
4
|
Summary: StepUp Core provides the basic framework for the StepUp build tool
|
|
5
5
|
Author-email: Toon Verstraelen <toon.verstraelen@ugent.be>
|
|
6
6
|
License-Expression: GPL-3.0-or-later
|
|
@@ -25,7 +25,8 @@ Requires-Python: >=3.11
|
|
|
25
25
|
Description-Content-Type: text/markdown
|
|
26
26
|
License-File: LICENSE
|
|
27
27
|
Requires-Dist: asyncinotify>=4.0.9
|
|
28
|
-
Requires-Dist: attrs>=
|
|
28
|
+
Requires-Dist: attrs>=23.1.0
|
|
29
|
+
Requires-Dist: cattrs>=23.2.3
|
|
29
30
|
Requires-Dist: jinja2>=3.1.6
|
|
30
31
|
Requires-Dist: parse>=1.19.1
|
|
31
32
|
Requires-Dist: path>=16.14.0
|
|
@@ -49,16 +50,53 @@ Dynamic: license-file
|
|
|
49
50
|
|
|
50
51
|
# StepUp Core
|
|
51
52
|
|
|
52
|
-
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
|
|
53
|
-
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
|
|
54
|
-
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
|
|
53
|
+
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
|
|
54
|
+
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
|
|
55
|
+
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
|
|
55
56
|
[](https://pypi.org/project/stepup/)
|
|
56
57
|

|
|
57
58
|

|
|
58
59
|
[](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
|
|
59
60
|
|
|
60
|
-
StepUp is a simple, powerful and universal build tool
|
|
61
|
+
StepUp is a simple, powerful and universal build tool,
|
|
62
|
+
a modern alternative to [Make](https://en.wikipedia.org/wiki/Make_(software)).
|
|
63
|
+
Its defining feature is that it treats the generation and execution of the build workflow
|
|
64
|
+
as one and the same thing:
|
|
65
|
+
a `plan.py` Python script issues build steps via RPC to a persistent background process,
|
|
66
|
+
so steps can be defined on the fly using the outputs of earlier steps.
|
|
67
|
+
This makes StepUp a good fit for builds where the full set of dependencies
|
|
68
|
+
cannot be known in advance.
|
|
69
|
+
|
|
61
70
|
StepUp Core provides the basic framework for StepUp, without any domain-specific features.
|
|
71
|
+
Those live in extension packages:
|
|
72
|
+
|
|
73
|
+
- [StepUp RepRep](https://reproducible-reporting.github.io/stepup-reprep/)
|
|
74
|
+
for creating **rep**roducible **rep**orts: papers, presentations, theses, etc.
|
|
75
|
+
- [StepUp Queue](https://reproducible-reporting.github.io/stepup-queue/)
|
|
76
|
+
submits jobs to a SLURM scheduler.
|
|
77
|
+
|
|
78
|
+
## Installation
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
pip install stepup
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
See the [installation guide](https://reproducible-reporting.github.io/stepup-core/installation/)
|
|
85
|
+
for details.
|
|
86
|
+
|
|
87
|
+
## Quick Visual Impression
|
|
88
|
+
|
|
89
|
+
[](https://asciinema.org/a/718833)
|
|
90
|
+
|
|
91
|
+
## Documentation
|
|
92
|
+
|
|
93
|
+
Full documentation, including a tutorial and a feature overview,
|
|
94
|
+
is available at <https://reproducible-reporting.github.io/stepup-core>.
|
|
95
|
+
|
|
96
|
+
## License
|
|
62
97
|
|
|
63
|
-
|
|
64
|
-
[
|
|
98
|
+
StepUp Core is distributed under the terms of the
|
|
99
|
+
[GPL-3.0-or-later](LICENSE) license.
|
|
100
|
+
Contributions are welcome;
|
|
101
|
+
see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
|
|
102
|
+
to get started.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
<!-- markdownlint-disable line-length -->
|
|
2
|
+
|
|
3
|
+
# StepUp Core
|
|
4
|
+
|
|
5
|
+
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/release.yaml)
|
|
6
|
+
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/pytest.yaml)
|
|
7
|
+
[](https://github.com/reproducible-reporting/stepup-core/actions/workflows/mkdocs.yaml)
|
|
8
|
+
[](https://pypi.org/project/stepup/)
|
|
9
|
+

|
|
10
|
+

|
|
11
|
+
[](https://www.codefactor.io/repository/github/reproducible-reporting/stepup-core)
|
|
12
|
+
|
|
13
|
+
StepUp is a simple, powerful and universal build tool,
|
|
14
|
+
a modern alternative to [Make](https://en.wikipedia.org/wiki/Make_(software)).
|
|
15
|
+
Its defining feature is that it treats the generation and execution of the build workflow
|
|
16
|
+
as one and the same thing:
|
|
17
|
+
a `plan.py` Python script issues build steps via RPC to a persistent background process,
|
|
18
|
+
so steps can be defined on the fly using the outputs of earlier steps.
|
|
19
|
+
This makes StepUp a good fit for builds where the full set of dependencies
|
|
20
|
+
cannot be known in advance.
|
|
21
|
+
|
|
22
|
+
StepUp Core provides the basic framework for StepUp, without any domain-specific features.
|
|
23
|
+
Those live in extension packages:
|
|
24
|
+
|
|
25
|
+
- [StepUp RepRep](https://reproducible-reporting.github.io/stepup-reprep/)
|
|
26
|
+
for creating **rep**roducible **rep**orts: papers, presentations, theses, etc.
|
|
27
|
+
- [StepUp Queue](https://reproducible-reporting.github.io/stepup-queue/)
|
|
28
|
+
submits jobs to a SLURM scheduler.
|
|
29
|
+
|
|
30
|
+
## Installation
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install stepup
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
See the [installation guide](https://reproducible-reporting.github.io/stepup-core/installation/)
|
|
37
|
+
for details.
|
|
38
|
+
|
|
39
|
+
## Quick Visual Impression
|
|
40
|
+
|
|
41
|
+
[](https://asciinema.org/a/718833)
|
|
42
|
+
|
|
43
|
+
## Documentation
|
|
44
|
+
|
|
45
|
+
Full documentation, including a tutorial and a feature overview,
|
|
46
|
+
is available at <https://reproducible-reporting.github.io/stepup-core>.
|
|
47
|
+
|
|
48
|
+
## License
|
|
49
|
+
|
|
50
|
+
StepUp Core is distributed under the terms of the
|
|
51
|
+
[GPL-3.0-or-later](LICENSE) license.
|
|
52
|
+
Contributions are welcome;
|
|
53
|
+
see the [development guide](https://reproducible-reporting.github.io/stepup-core/development/)
|
|
54
|
+
to get started.
|
|
@@ -29,7 +29,8 @@ classifiers = [
|
|
|
29
29
|
dependencies = [
|
|
30
30
|
# Ensure changes to these dependencies are reflected in .github/requirements-old.txt
|
|
31
31
|
"asyncinotify>=4.0.9",
|
|
32
|
-
"attrs>=
|
|
32
|
+
"attrs>=23.1.0",
|
|
33
|
+
"cattrs>=23.2.3",
|
|
33
34
|
"jinja2>=3.1.6",
|
|
34
35
|
"parse>=1.19.1",
|
|
35
36
|
"path>=16.14.0",
|
|
@@ -61,11 +62,14 @@ Changelog = "https://reproducible-reporting.github.io/stepup-core/changelog/"
|
|
|
61
62
|
|
|
62
63
|
[project.scripts]
|
|
63
64
|
stepup = "stepup.core.__main__:main"
|
|
65
|
+
sb = "stepup.core.__main__:sb_main"
|
|
66
|
+
_stepup_hasher = "stepup.core.hasher:hasher_tool"
|
|
67
|
+
render-jinja = "stepup.core.render_jinja:main"
|
|
64
68
|
|
|
65
69
|
[project.entry-points."stepup.tools"]
|
|
66
|
-
act = "stepup.core.actions:act_subcommand"
|
|
67
|
-
browse = "stepup.core.browse:browse_subcommand"
|
|
68
70
|
boot = "stepup.core.tui:boot_subcommand"
|
|
71
|
+
browse = "stepup.core.browse:browse_subcommand"
|
|
72
|
+
build = "stepup.core.tui:build_subcommand"
|
|
69
73
|
clean = "stepup.core.clean:clean_subcommand"
|
|
70
74
|
drain = "stepup.core.interact:drain_subcommand"
|
|
71
75
|
run = "stepup.core.interact:run_subcommand"
|
|
@@ -74,20 +78,18 @@ watch-update = "stepup.core.interact:watch_update_subcommand"
|
|
|
74
78
|
watch-delete = "stepup.core.interact:watch_delete_subcommand"
|
|
75
79
|
wait = "stepup.core.interact:wait_subcommand"
|
|
76
80
|
join = "stepup.core.interact:join_subcommand"
|
|
77
|
-
|
|
81
|
+
show-config = "stepup.core.config:show_config_subcommand"
|
|
78
82
|
shutdown = "stepup.core.interact:shutdown_subcommand"
|
|
79
83
|
status = "stepup.core.interact:status_subcommand"
|
|
80
84
|
|
|
81
|
-
[project.entry-points."stepup.actions"]
|
|
82
|
-
copy = "stepup.core.actions:copy"
|
|
83
|
-
mkdir = "stepup.core.actions:mkdir"
|
|
84
|
-
render-jinja = "stepup.core.render_jinja:render_jinja_action"
|
|
85
|
-
runpy = "stepup.core.actions:runpy"
|
|
86
|
-
runsh = "stepup.core.actions:runsh"
|
|
87
85
|
|
|
88
86
|
[tool.pytest.ini_options]
|
|
89
87
|
addopts = "-n auto --dist worksteal -W error -W ignore::ResourceWarning"
|
|
88
|
+
asyncio_mode = "auto"
|
|
90
89
|
asyncio_default_fixture_loop_scope = "function"
|
|
90
|
+
markers = [
|
|
91
|
+
"requires_fork_runpy: mark test that requires STEPUP_BUILD_FORK_RUNPY != 0",
|
|
92
|
+
]
|
|
91
93
|
|
|
92
94
|
[tool.ruff]
|
|
93
95
|
line-length = 100
|
|
@@ -27,6 +27,7 @@ from importlib.metadata import version as get_version
|
|
|
27
27
|
|
|
28
28
|
from path import Path
|
|
29
29
|
|
|
30
|
+
from .config import ConfigLoader
|
|
30
31
|
from .utils import string_to_bool
|
|
31
32
|
|
|
32
33
|
|
|
@@ -40,7 +41,27 @@ def main():
|
|
|
40
41
|
parser.print_help()
|
|
41
42
|
|
|
42
43
|
|
|
44
|
+
def sb_main():
|
|
45
|
+
"""Shortcut for `stepup build` (accepts the same arguments)."""
|
|
46
|
+
sys.argv.insert(1, "build")
|
|
47
|
+
main()
|
|
48
|
+
|
|
49
|
+
|
|
43
50
|
def build_parser() -> tuple[argparse.ArgumentParser, list[callable]]:
|
|
51
|
+
# Configuration loader
|
|
52
|
+
stepup_root = Path(os.getenv("STEPUP_ROOT", os.getcwd()))
|
|
53
|
+
loader = ConfigLoader(
|
|
54
|
+
prefix="stepup",
|
|
55
|
+
config_paths=[
|
|
56
|
+
"/etc/stepup.toml",
|
|
57
|
+
"~/.config/stepup.toml",
|
|
58
|
+
stepup_root / ".stepup.toml",
|
|
59
|
+
stepup_root / "stepup.toml",
|
|
60
|
+
stepup_root / "pyproject.toml",
|
|
61
|
+
stepup_root / "stepup-local.toml",
|
|
62
|
+
],
|
|
63
|
+
)
|
|
64
|
+
|
|
44
65
|
# Base argument parser
|
|
45
66
|
parser = argparse.ArgumentParser(
|
|
46
67
|
prog="stepup",
|
|
@@ -56,12 +77,7 @@ def build_parser() -> tuple[argparse.ArgumentParser, list[callable]]:
|
|
|
56
77
|
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
|
57
78
|
help="Set the logging level. [default=%(default)s]",
|
|
58
79
|
)
|
|
59
|
-
|
|
60
|
-
"--root",
|
|
61
|
-
type=Path,
|
|
62
|
-
default=Path(os.getenv("STEPUP_ROOT", os.getcwd())),
|
|
63
|
-
help="Directory containing top-level plan.py [default=%(default)s]",
|
|
64
|
-
)
|
|
80
|
+
loader.patch_parser(parser, use_section=False)
|
|
65
81
|
|
|
66
82
|
# Load tool entry points
|
|
67
83
|
subparsers = parser.add_subparsers(dest="tool", required=False)
|
|
@@ -69,7 +85,7 @@ def build_parser() -> tuple[argparse.ArgumentParser, list[callable]]:
|
|
|
69
85
|
tool_funcs = {}
|
|
70
86
|
for tool_ep in tool_eps:
|
|
71
87
|
tool = tool_ep.load()
|
|
72
|
-
tool_funcs[tool_ep.name] = tool(subparsers)
|
|
88
|
+
tool_funcs[tool_ep.name] = tool(subparsers, loader)
|
|
73
89
|
|
|
74
90
|
return parser, tool_funcs
|
|
75
91
|
|