slurm-avail 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slurm_avail-0.1.0/.github/workflows/ci.yml +42 -0
- slurm_avail-0.1.0/.github/workflows/release.yml +43 -0
- slurm_avail-0.1.0/.gitignore +8 -0
- slurm_avail-0.1.0/LICENSE +21 -0
- slurm_avail-0.1.0/PKG-INFO +274 -0
- slurm_avail-0.1.0/README.md +247 -0
- slurm_avail-0.1.0/RELEASING.md +25 -0
- slurm_avail-0.1.0/assets/nodes-dashboard.png +0 -0
- slurm_avail-0.1.0/assets/scheduler-forecast.png +0 -0
- slurm_avail-0.1.0/assets/slurm-avail-logo.png +0 -0
- slurm_avail-0.1.0/config.example.toml +36 -0
- slurm_avail-0.1.0/pyproject.toml +53 -0
- slurm_avail-0.1.0/src/slurm_avail/__init__.py +12 -0
- slurm_avail-0.1.0/src/slurm_avail/__main__.py +5 -0
- slurm_avail-0.1.0/src/slurm_avail/cli.py +145 -0
- slurm_avail-0.1.0/src/slurm_avail/collect.py +779 -0
- slurm_avail-0.1.0/src/slurm_avail/config.py +305 -0
- slurm_avail-0.1.0/src/slurm_avail/config_views.py +252 -0
- slurm_avail-0.1.0/src/slurm_avail/constants.py +28 -0
- slurm_avail-0.1.0/src/slurm_avail/forecast_views.py +492 -0
- slurm_avail-0.1.0/src/slurm_avail/models.py +98 -0
- slurm_avail-0.1.0/src/slurm_avail/node_views.py +442 -0
- slurm_avail-0.1.0/src/slurm_avail/output.py +194 -0
- slurm_avail-0.1.0/src/slurm_avail/text.py +118 -0
- slurm_avail-0.1.0/src/slurm_avail/tui.py +1018 -0
- slurm_avail-0.1.0/tests/test_collect.py +52 -0
- slurm_avail-0.1.0/tests/test_config.py +46 -0
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
test:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
python-version: ["3.11", "3.14"]
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v6
|
|
20
|
+
- uses: actions/setup-python@v7
|
|
21
|
+
with:
|
|
22
|
+
python-version: ${{ matrix.python-version }}
|
|
23
|
+
- name: Install project
|
|
24
|
+
run: python -m pip install --upgrade --editable '.[dev]'
|
|
25
|
+
- name: Lint
|
|
26
|
+
run: |
|
|
27
|
+
ruff format --check .
|
|
28
|
+
ruff check .
|
|
29
|
+
- name: Test
|
|
30
|
+
run: pytest
|
|
31
|
+
|
|
32
|
+
build:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
steps:
|
|
35
|
+
- uses: actions/checkout@v6
|
|
36
|
+
- uses: actions/setup-python@v7
|
|
37
|
+
with:
|
|
38
|
+
python-version: "3.13"
|
|
39
|
+
- name: Build distributions
|
|
40
|
+
run: |
|
|
41
|
+
python -m pip install build
|
|
42
|
+
python -m build
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: Publish Python package
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
build:
|
|
12
|
+
runs-on: ubuntu-latest
|
|
13
|
+
steps:
|
|
14
|
+
- uses: actions/checkout@v6
|
|
15
|
+
- uses: actions/setup-python@v7
|
|
16
|
+
with:
|
|
17
|
+
python-version: "3.13"
|
|
18
|
+
- name: Build release distributions
|
|
19
|
+
run: |
|
|
20
|
+
python -m pip install build
|
|
21
|
+
python -m build
|
|
22
|
+
- name: Upload distributions
|
|
23
|
+
uses: actions/upload-artifact@v4
|
|
24
|
+
with:
|
|
25
|
+
name: release-distributions
|
|
26
|
+
path: dist/
|
|
27
|
+
|
|
28
|
+
publish:
|
|
29
|
+
needs: build
|
|
30
|
+
runs-on: ubuntu-latest
|
|
31
|
+
environment:
|
|
32
|
+
name: pypi
|
|
33
|
+
url: https://pypi.org/p/slurm-avail
|
|
34
|
+
permissions:
|
|
35
|
+
id-token: write
|
|
36
|
+
steps:
|
|
37
|
+
- name: Download distributions
|
|
38
|
+
uses: actions/download-artifact@v5
|
|
39
|
+
with:
|
|
40
|
+
name: release-distributions
|
|
41
|
+
path: dist/
|
|
42
|
+
- name: Publish to PyPI
|
|
43
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 slurm-avail contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: slurm-avail
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A configurable terminal dashboard for Slurm cluster availability
|
|
5
|
+
Project-URL: Homepage, https://github.com/LukasBuschmann/slurm-avail
|
|
6
|
+
Project-URL: Repository, https://github.com/LukasBuschmann/slurm-avail
|
|
7
|
+
Project-URL: Issues, https://github.com/LukasBuschmann/slurm-avail/issues
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: GPU,HPC,Slurm,TUI,cluster
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Environment :: Console :: Curses
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Operating System :: POSIX
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: System :: Monitoring
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: build>=1.2; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff>=0.11; extra == 'dev'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
<p align="center">
|
|
29
|
+
<img src="assets/slurm-avail-logo.png" alt="slurm-avail" width="680">
|
|
30
|
+
</p>
|
|
31
|
+
|
|
32
|
+
<p align="center">
|
|
33
|
+
A live, configurable terminal dashboard for multiple Slurm clusters.
|
|
34
|
+
</p>
|
|
35
|
+
|
|
36
|
+
<p align="center">
|
|
37
|
+
<a href="https://github.com/LukasBuschmann/slurm-avail/actions/workflows/ci.yml"><img src="https://github.com/LukasBuschmann/slurm-avail/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
38
|
+
<img src="https://img.shields.io/badge/python-%3E%3D3.11-3776AB" alt="Python 3.11 or newer">
|
|
39
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license"></a>
|
|
40
|
+
</p>
|
|
41
|
+
|
|
42
|
+
`slurm-avail` brings live CPU, memory, GPU, filesystem, and scheduler data from
|
|
43
|
+
multiple Slurm clusters into one keyboard-driven TUI. Clusters can be queried
|
|
44
|
+
locally or over SSH. No `slurm-avail` agent or service needs to be installed on
|
|
45
|
+
the HPC systems; the dashboard uses their existing Slurm command-line tools.
|
|
46
|
+
|
|
47
|
+
| Live multi-cluster overview | Scheduler forecast |
|
|
48
|
+
| --- | --- |
|
|
49
|
+
|  |  |
|
|
50
|
+
|
|
51
|
+
## Views
|
|
52
|
+
|
|
53
|
+
**Nodes** gives a live overview of all configured clusters. It summarizes free
|
|
54
|
+
CPU, memory, and physical GPU capacity and then shows how those resources are
|
|
55
|
+
distributed across individual nodes. Node states such as allocated, reserved,
|
|
56
|
+
user-exclusive, drained, and down remain visible in the same view.
|
|
57
|
+
|
|
58
|
+
**Filesystems** compares capacity and available space for the storage paths
|
|
59
|
+
configured on each cluster.
|
|
60
|
+
|
|
61
|
+
**Login Nodes** shows which SSH endpoints are reachable, which endpoint is
|
|
62
|
+
currently supplying data, and whether automatic failover is available.
|
|
63
|
+
|
|
64
|
+
**Forecast** presents a per-node timeline of exact Slurm reservations and the
|
|
65
|
+
remaining time limits of currently running jobs. The timeline can be zoomed
|
|
66
|
+
and navigated from short operational windows up to a year. It deliberately
|
|
67
|
+
does not speculate about when pending jobs will start.
|
|
68
|
+
|
|
69
|
+
**Config** manages clusters and dashboard behavior without leaving the TUI.
|
|
70
|
+
Clusters can use local or SSH collection, multiple login endpoints, CPU or GPU
|
|
71
|
+
focused layouts, custom Slurm locations, and selected filesystem paths. They
|
|
72
|
+
can also be hidden or reordered, while refresh and retry intervals are shared
|
|
73
|
+
dashboard settings. Everything is persisted in a readable TOML file.
|
|
74
|
+
|
|
75
|
+
## Installation
|
|
76
|
+
|
|
77
|
+
Install the current GitHub version with [`uv`](https://docs.astral.sh/uv/):
|
|
78
|
+
|
|
79
|
+
```console
|
|
80
|
+
uv tool install git+https://github.com/LukasBuschmann/slurm-avail.git
|
|
81
|
+
slurm-avail
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
After the first PyPI release, installation is simply:
|
|
85
|
+
|
|
86
|
+
```console
|
|
87
|
+
uv tool install slurm-avail
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
[`pipx`](https://pipx.pypa.io/) is also supported:
|
|
91
|
+
|
|
92
|
+
```console
|
|
93
|
+
pipx install slurm-avail
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
The package has no third-party runtime dependencies.
|
|
97
|
+
|
|
98
|
+
## Quick start
|
|
99
|
+
|
|
100
|
+
Run:
|
|
101
|
+
|
|
102
|
+
```console
|
|
103
|
+
slurm-avail
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
On first launch, the Config view opens automatically and creates:
|
|
107
|
+
|
|
108
|
+
```text
|
|
109
|
+
~/.config/slurm-avail/config.toml
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Select the initial local cluster and press `Enter` to edit it, or press `a` to
|
|
113
|
+
add an SSH cluster. Press `s` to save and apply, then use `Tab` to switch
|
|
114
|
+
between Nodes, Filesystems, Login Nodes, Forecast, and Config.
|
|
115
|
+
|
|
116
|
+
You can return directly to configuration with:
|
|
117
|
+
|
|
118
|
+
```console
|
|
119
|
+
slurm-avail --init
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
## Configuration
|
|
123
|
+
|
|
124
|
+
Clusters may run locally or be reached through one or more SSH endpoints.
|
|
125
|
+
|
|
126
|
+
The following is an example configuration for the **TU Dresden HPC system**.
|
|
127
|
+
It monitors the Capella, Alpha, Barnard, and Romeo clusters, including their
|
|
128
|
+
two login nodes and shared filesystems. Replace `USERNAME` with your own
|
|
129
|
+
TU Dresden/ZIH username.
|
|
130
|
+
|
|
131
|
+
<details>
|
|
132
|
+
<summary>Show TU Dresden HPC example configuration</summary>
|
|
133
|
+
|
|
134
|
+
```toml
|
|
135
|
+
version = 1
|
|
136
|
+
|
|
137
|
+
[settings]
|
|
138
|
+
node_refresh_seconds = 10
|
|
139
|
+
filesystem_refresh_seconds = 60
|
|
140
|
+
login_refresh_seconds = 60
|
|
141
|
+
forecast_refresh_seconds = 300
|
|
142
|
+
failed_retry_seconds = 10
|
|
143
|
+
ssh_connect_timeout_seconds = 8
|
|
144
|
+
command_timeout_seconds = 20
|
|
145
|
+
login_probe_timeout_seconds = 10
|
|
146
|
+
filesystem_probe_timeout_seconds = 2
|
|
147
|
+
retries_per_address = 0
|
|
148
|
+
retry_delay_seconds = 1
|
|
149
|
+
forecast_horizon_days = 365
|
|
150
|
+
|
|
151
|
+
[[clusters]]
|
|
152
|
+
name = "CAPELLA"
|
|
153
|
+
mode = "ssh"
|
|
154
|
+
addresses = ["login1.capella.hpc.tu-dresden.de", "login2.capella.hpc.tu-dresden.de"]
|
|
155
|
+
user = "USERNAME"
|
|
156
|
+
focus = "gpu"
|
|
157
|
+
hidden = false
|
|
158
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
159
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
160
|
+
exclude_partitions = ["interactive", "capella-interactive"]
|
|
161
|
+
|
|
162
|
+
[[clusters]]
|
|
163
|
+
name = "ALPHA"
|
|
164
|
+
mode = "ssh"
|
|
165
|
+
addresses = ["login1.alpha.hpc.tu-dresden.de", "login2.alpha.hpc.tu-dresden.de"]
|
|
166
|
+
user = "USERNAME"
|
|
167
|
+
focus = "gpu"
|
|
168
|
+
hidden = false
|
|
169
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
170
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
171
|
+
exclude_partitions = ["interactive", "alpha-interactive"]
|
|
172
|
+
|
|
173
|
+
[[clusters]]
|
|
174
|
+
name = "BARNARD"
|
|
175
|
+
mode = "ssh"
|
|
176
|
+
addresses = ["login1.barnard.hpc.tu-dresden.de", "login2.barnard.hpc.tu-dresden.de"]
|
|
177
|
+
user = "USERNAME"
|
|
178
|
+
focus = "cpu"
|
|
179
|
+
hidden = false
|
|
180
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
181
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
182
|
+
exclude_partitions = ["interactive"]
|
|
183
|
+
|
|
184
|
+
[[clusters]]
|
|
185
|
+
name = "ROMEO"
|
|
186
|
+
mode = "ssh"
|
|
187
|
+
addresses = ["login1.romeo.hpc.tu-dresden.de", "login2.romeo.hpc.tu-dresden.de"]
|
|
188
|
+
user = "USERNAME"
|
|
189
|
+
focus = "cpu"
|
|
190
|
+
hidden = false
|
|
191
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
192
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
193
|
+
exclude_partitions = ["interactive"]
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
</details>
|
|
197
|
+
|
|
198
|
+
For a smaller generic example, see [config.example.toml](config.example.toml).
|
|
199
|
+
The `user` field is optional; when omitted, OpenSSH configuration determines
|
|
200
|
+
the user. The Slurm path is also optional when `scontrol` and `squeue` are
|
|
201
|
+
already available on the endpoint's `PATH`.
|
|
202
|
+
|
|
203
|
+
A different configuration file can be selected with:
|
|
204
|
+
|
|
205
|
+
```console
|
|
206
|
+
slurm-avail --config /path/to/config.toml
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
`XDG_CONFIG_HOME` is respected.
|
|
210
|
+
|
|
211
|
+
## Controls
|
|
212
|
+
|
|
213
|
+
| Key | Action |
|
|
214
|
+
| --- | --- |
|
|
215
|
+
| `Tab` | Switch view |
|
|
216
|
+
| Arrow keys | Scroll or navigate |
|
|
217
|
+
| `Page Up` / `Page Down` | Scroll by page |
|
|
218
|
+
| `Home` / `End` | Jump to top or bottom |
|
|
219
|
+
| `r` | Refresh now |
|
|
220
|
+
| `+` / `-` | Change forecast resolution |
|
|
221
|
+
| `[` / `]` | Change forecast cluster |
|
|
222
|
+
| `q` | Quit |
|
|
223
|
+
|
|
224
|
+
The Config view shows its editing controls in the legend. In particular, use
|
|
225
|
+
`a` to add, `Enter` to edit, `h` to hide or show, `Shift+Up/Down` to reorder,
|
|
226
|
+
and `s` to save clusters.
|
|
227
|
+
|
|
228
|
+
## Requirements
|
|
229
|
+
|
|
230
|
+
On the computer running `slurm-avail`:
|
|
231
|
+
|
|
232
|
+
- Python 3.11 or newer
|
|
233
|
+
- a curses-capable POSIX terminal
|
|
234
|
+
- OpenSSH's `ssh` command for remote clusters
|
|
235
|
+
|
|
236
|
+
On each configured cluster endpoint:
|
|
237
|
+
|
|
238
|
+
- a working Slurm installation connected to the cluster controller
|
|
239
|
+
- Slurm's `scontrol` and `squeue` commands available on `PATH` or through the
|
|
240
|
+
configured `slurm_bin_path`
|
|
241
|
+
- `/bin/sh`, `id`, and `date`
|
|
242
|
+
- `df` and `tail` for filesystem monitoring (`timeout` is used when present)
|
|
243
|
+
|
|
244
|
+
SSH authentication must already work non-interactively, normally through an
|
|
245
|
+
SSH agent, keys, and `~/.ssh/config`. `slurm-avail` does not store passwords or
|
|
246
|
+
private keys.
|
|
247
|
+
|
|
248
|
+
## Command-line usage
|
|
249
|
+
|
|
250
|
+
```console
|
|
251
|
+
slurm-avail --help
|
|
252
|
+
slurm-avail --version
|
|
253
|
+
slurm-avail --once
|
|
254
|
+
slurm-avail --view forecast --cluster 2
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
## Development
|
|
258
|
+
|
|
259
|
+
```console
|
|
260
|
+
git clone https://github.com/LukasBuschmann/slurm-avail.git
|
|
261
|
+
cd slurm-avail
|
|
262
|
+
python -m venv .venv
|
|
263
|
+
. .venv/bin/activate
|
|
264
|
+
python -m pip install -e '.[dev]'
|
|
265
|
+
pytest
|
|
266
|
+
ruff check .
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
Contributions and issue reports are welcome. Release instructions are in
|
|
270
|
+
[RELEASING.md](RELEASING.md).
|
|
271
|
+
|
|
272
|
+
## License
|
|
273
|
+
|
|
274
|
+
`slurm-avail` is available under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="assets/slurm-avail-logo.png" alt="slurm-avail" width="680">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
<p align="center">
|
|
6
|
+
A live, configurable terminal dashboard for multiple Slurm clusters.
|
|
7
|
+
</p>
|
|
8
|
+
|
|
9
|
+
<p align="center">
|
|
10
|
+
<a href="https://github.com/LukasBuschmann/slurm-avail/actions/workflows/ci.yml"><img src="https://github.com/LukasBuschmann/slurm-avail/actions/workflows/ci.yml/badge.svg" alt="CI"></a>
|
|
11
|
+
<img src="https://img.shields.io/badge/python-%3E%3D3.11-3776AB" alt="Python 3.11 or newer">
|
|
12
|
+
<a href="LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="MIT license"></a>
|
|
13
|
+
</p>
|
|
14
|
+
|
|
15
|
+
`slurm-avail` brings live CPU, memory, GPU, filesystem, and scheduler data from
|
|
16
|
+
multiple Slurm clusters into one keyboard-driven TUI. Clusters can be queried
|
|
17
|
+
locally or over SSH. No `slurm-avail` agent or service needs to be installed on
|
|
18
|
+
the HPC systems; the dashboard uses their existing Slurm command-line tools.
|
|
19
|
+
|
|
20
|
+
| Live multi-cluster overview | Scheduler forecast |
|
|
21
|
+
| --- | --- |
|
|
22
|
+
|  |  |
|
|
23
|
+
|
|
24
|
+
## Views
|
|
25
|
+
|
|
26
|
+
**Nodes** gives a live overview of all configured clusters. It summarizes free
|
|
27
|
+
CPU, memory, and physical GPU capacity and then shows how those resources are
|
|
28
|
+
distributed across individual nodes. Node states such as allocated, reserved,
|
|
29
|
+
user-exclusive, drained, and down remain visible in the same view.
|
|
30
|
+
|
|
31
|
+
**Filesystems** compares capacity and available space for the storage paths
|
|
32
|
+
configured on each cluster.
|
|
33
|
+
|
|
34
|
+
**Login Nodes** shows which SSH endpoints are reachable, which endpoint is
|
|
35
|
+
currently supplying data, and whether automatic failover is available.
|
|
36
|
+
|
|
37
|
+
**Forecast** presents a per-node timeline of exact Slurm reservations and the
|
|
38
|
+
remaining time limits of currently running jobs. The timeline can be zoomed
|
|
39
|
+
and navigated from short operational windows up to a year. It deliberately
|
|
40
|
+
does not speculate about when pending jobs will start.
|
|
41
|
+
|
|
42
|
+
**Config** manages clusters and dashboard behavior without leaving the TUI.
|
|
43
|
+
Clusters can use local or SSH collection, multiple login endpoints, CPU or GPU
|
|
44
|
+
focused layouts, custom Slurm locations, and selected filesystem paths. They
|
|
45
|
+
can also be hidden or reordered, while refresh and retry intervals are shared
|
|
46
|
+
dashboard settings. Everything is persisted in a readable TOML file.
|
|
47
|
+
|
|
48
|
+
## Installation
|
|
49
|
+
|
|
50
|
+
Install the current GitHub version with [`uv`](https://docs.astral.sh/uv/):
|
|
51
|
+
|
|
52
|
+
```console
|
|
53
|
+
uv tool install git+https://github.com/LukasBuschmann/slurm-avail.git
|
|
54
|
+
slurm-avail
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
After the first PyPI release, installation is simply:
|
|
58
|
+
|
|
59
|
+
```console
|
|
60
|
+
uv tool install slurm-avail
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
[`pipx`](https://pipx.pypa.io/) is also supported:
|
|
64
|
+
|
|
65
|
+
```console
|
|
66
|
+
pipx install slurm-avail
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The package has no third-party runtime dependencies.
|
|
70
|
+
|
|
71
|
+
## Quick start
|
|
72
|
+
|
|
73
|
+
Run:
|
|
74
|
+
|
|
75
|
+
```console
|
|
76
|
+
slurm-avail
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
On first launch, the Config view opens automatically and creates:
|
|
80
|
+
|
|
81
|
+
```text
|
|
82
|
+
~/.config/slurm-avail/config.toml
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
Select the initial local cluster and press `Enter` to edit it, or press `a` to
|
|
86
|
+
add an SSH cluster. Press `s` to save and apply, then use `Tab` to switch
|
|
87
|
+
between Nodes, Filesystems, Login Nodes, Forecast, and Config.
|
|
88
|
+
|
|
89
|
+
You can return directly to configuration with:
|
|
90
|
+
|
|
91
|
+
```console
|
|
92
|
+
slurm-avail --init
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Configuration
|
|
96
|
+
|
|
97
|
+
Clusters may run locally or be reached through one or more SSH endpoints.
|
|
98
|
+
|
|
99
|
+
The following is an example configuration for the **TU Dresden HPC system**.
|
|
100
|
+
It monitors the Capella, Alpha, Barnard, and Romeo clusters, including their
|
|
101
|
+
two login nodes and shared filesystems. Replace `USERNAME` with your own
|
|
102
|
+
TU Dresden/ZIH username.
|
|
103
|
+
|
|
104
|
+
<details>
|
|
105
|
+
<summary>Show TU Dresden HPC example configuration</summary>
|
|
106
|
+
|
|
107
|
+
```toml
|
|
108
|
+
version = 1
|
|
109
|
+
|
|
110
|
+
[settings]
|
|
111
|
+
node_refresh_seconds = 10
|
|
112
|
+
filesystem_refresh_seconds = 60
|
|
113
|
+
login_refresh_seconds = 60
|
|
114
|
+
forecast_refresh_seconds = 300
|
|
115
|
+
failed_retry_seconds = 10
|
|
116
|
+
ssh_connect_timeout_seconds = 8
|
|
117
|
+
command_timeout_seconds = 20
|
|
118
|
+
login_probe_timeout_seconds = 10
|
|
119
|
+
filesystem_probe_timeout_seconds = 2
|
|
120
|
+
retries_per_address = 0
|
|
121
|
+
retry_delay_seconds = 1
|
|
122
|
+
forecast_horizon_days = 365
|
|
123
|
+
|
|
124
|
+
[[clusters]]
|
|
125
|
+
name = "CAPELLA"
|
|
126
|
+
mode = "ssh"
|
|
127
|
+
addresses = ["login1.capella.hpc.tu-dresden.de", "login2.capella.hpc.tu-dresden.de"]
|
|
128
|
+
user = "USERNAME"
|
|
129
|
+
focus = "gpu"
|
|
130
|
+
hidden = false
|
|
131
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
132
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
133
|
+
exclude_partitions = ["interactive", "capella-interactive"]
|
|
134
|
+
|
|
135
|
+
[[clusters]]
|
|
136
|
+
name = "ALPHA"
|
|
137
|
+
mode = "ssh"
|
|
138
|
+
addresses = ["login1.alpha.hpc.tu-dresden.de", "login2.alpha.hpc.tu-dresden.de"]
|
|
139
|
+
user = "USERNAME"
|
|
140
|
+
focus = "gpu"
|
|
141
|
+
hidden = false
|
|
142
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
143
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
144
|
+
exclude_partitions = ["interactive", "alpha-interactive"]
|
|
145
|
+
|
|
146
|
+
[[clusters]]
|
|
147
|
+
name = "BARNARD"
|
|
148
|
+
mode = "ssh"
|
|
149
|
+
addresses = ["login1.barnard.hpc.tu-dresden.de", "login2.barnard.hpc.tu-dresden.de"]
|
|
150
|
+
user = "USERNAME"
|
|
151
|
+
focus = "cpu"
|
|
152
|
+
hidden = false
|
|
153
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
154
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
155
|
+
exclude_partitions = ["interactive"]
|
|
156
|
+
|
|
157
|
+
[[clusters]]
|
|
158
|
+
name = "ROMEO"
|
|
159
|
+
mode = "ssh"
|
|
160
|
+
addresses = ["login1.romeo.hpc.tu-dresden.de", "login2.romeo.hpc.tu-dresden.de"]
|
|
161
|
+
user = "USERNAME"
|
|
162
|
+
focus = "cpu"
|
|
163
|
+
hidden = false
|
|
164
|
+
filesystems = ["/home", "/software", "/data/horse", "/data/walrus", "/data/narwhal", "/data/quokka", "/data/cat"]
|
|
165
|
+
slurm_bin_path = "/opt/slurm/current/bin"
|
|
166
|
+
exclude_partitions = ["interactive"]
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
</details>
|
|
170
|
+
|
|
171
|
+
For a smaller generic example, see [config.example.toml](config.example.toml).
|
|
172
|
+
The `user` field is optional; when omitted, OpenSSH configuration determines
|
|
173
|
+
the user. The Slurm path is also optional when `scontrol` and `squeue` are
|
|
174
|
+
already available on the endpoint's `PATH`.
|
|
175
|
+
|
|
176
|
+
A different configuration file can be selected with:
|
|
177
|
+
|
|
178
|
+
```console
|
|
179
|
+
slurm-avail --config /path/to/config.toml
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
`XDG_CONFIG_HOME` is respected.
|
|
183
|
+
|
|
184
|
+
## Controls
|
|
185
|
+
|
|
186
|
+
| Key | Action |
|
|
187
|
+
| --- | --- |
|
|
188
|
+
| `Tab` | Switch view |
|
|
189
|
+
| Arrow keys | Scroll or navigate |
|
|
190
|
+
| `Page Up` / `Page Down` | Scroll by page |
|
|
191
|
+
| `Home` / `End` | Jump to top or bottom |
|
|
192
|
+
| `r` | Refresh now |
|
|
193
|
+
| `+` / `-` | Change forecast resolution |
|
|
194
|
+
| `[` / `]` | Change forecast cluster |
|
|
195
|
+
| `q` | Quit |
|
|
196
|
+
|
|
197
|
+
The Config view shows its editing controls in the legend. In particular, use
|
|
198
|
+
`a` to add, `Enter` to edit, `h` to hide or show, `Shift+Up/Down` to reorder,
|
|
199
|
+
and `s` to save clusters.
|
|
200
|
+
|
|
201
|
+
## Requirements
|
|
202
|
+
|
|
203
|
+
On the computer running `slurm-avail`:
|
|
204
|
+
|
|
205
|
+
- Python 3.11 or newer
|
|
206
|
+
- a curses-capable POSIX terminal
|
|
207
|
+
- OpenSSH's `ssh` command for remote clusters
|
|
208
|
+
|
|
209
|
+
On each configured cluster endpoint:
|
|
210
|
+
|
|
211
|
+
- a working Slurm installation connected to the cluster controller
|
|
212
|
+
- Slurm's `scontrol` and `squeue` commands available on `PATH` or through the
|
|
213
|
+
configured `slurm_bin_path`
|
|
214
|
+
- `/bin/sh`, `id`, and `date`
|
|
215
|
+
- `df` and `tail` for filesystem monitoring (`timeout` is used when present)
|
|
216
|
+
|
|
217
|
+
SSH authentication must already work non-interactively, normally through an
|
|
218
|
+
SSH agent, keys, and `~/.ssh/config`. `slurm-avail` does not store passwords or
|
|
219
|
+
private keys.
|
|
220
|
+
|
|
221
|
+
## Command-line usage
|
|
222
|
+
|
|
223
|
+
```console
|
|
224
|
+
slurm-avail --help
|
|
225
|
+
slurm-avail --version
|
|
226
|
+
slurm-avail --once
|
|
227
|
+
slurm-avail --view forecast --cluster 2
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
## Development
|
|
231
|
+
|
|
232
|
+
```console
|
|
233
|
+
git clone https://github.com/LukasBuschmann/slurm-avail.git
|
|
234
|
+
cd slurm-avail
|
|
235
|
+
python -m venv .venv
|
|
236
|
+
. .venv/bin/activate
|
|
237
|
+
python -m pip install -e '.[dev]'
|
|
238
|
+
pytest
|
|
239
|
+
ruff check .
|
|
240
|
+
```
|
|
241
|
+
|
|
242
|
+
Contributions and issue reports are welcome. Release instructions are in
|
|
243
|
+
[RELEASING.md](RELEASING.md).
|
|
244
|
+
|
|
245
|
+
## License
|
|
246
|
+
|
|
247
|
+
`slurm-avail` is available under the [MIT License](LICENSE).
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Releasing
|
|
2
|
+
|
|
3
|
+
Releases are published to PyPI from GitHub Actions with Trusted Publishing, so
|
|
4
|
+
the repository does not need to contain a PyPI password or API token.
|
|
5
|
+
|
|
6
|
+
## One-time setup
|
|
7
|
+
|
|
8
|
+
1. Create the public `LukasBuschmann/slurm-avail` GitHub repository.
|
|
9
|
+
2. Create a `pypi` environment in the repository settings. Requiring approval
|
|
10
|
+
for that environment is recommended.
|
|
11
|
+
3. On PyPI, add a pending Trusted Publisher for project `slurm-avail` with:
|
|
12
|
+
- Owner: `LukasBuschmann`
|
|
13
|
+
- Repository: `slurm-avail`
|
|
14
|
+
- Workflow: `release.yml`
|
|
15
|
+
- Environment: `pypi`
|
|
16
|
+
|
|
17
|
+
## Publish a version
|
|
18
|
+
|
|
19
|
+
1. Update the version in `pyproject.toml`.
|
|
20
|
+
2. Run `ruff format --check .`, `ruff check .`, `pytest`, and `python -m build`.
|
|
21
|
+
3. Commit the version change and create a GitHub release tagged `vX.Y.Z`.
|
|
22
|
+
|
|
23
|
+
Publishing the GitHub release starts `.github/workflows/release.yml`. It builds
|
|
24
|
+
both the source distribution and wheel, then publishes them to PyPI after the
|
|
25
|
+
`pypi` environment has approved the job.
|
|
Binary file
|
|
Binary file
|
|
Binary file
|