slurmscope 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- slurmscope-1.2.0/LICENSE +21 -0
- slurmscope-1.2.0/MANIFEST.in +3 -0
- slurmscope-1.2.0/PKG-INFO +215 -0
- slurmscope-1.2.0/README.md +189 -0
- slurmscope-1.2.0/pyproject.toml +47 -0
- slurmscope-1.2.0/setup.cfg +4 -0
- slurmscope-1.2.0/src/slurmscope/__init__.py +3 -0
- slurmscope-1.2.0/src/slurmscope/__main__.py +6 -0
- slurmscope-1.2.0/src/slurmscope/cli.py +2833 -0
- slurmscope-1.2.0/src/slurmscope.egg-info/PKG-INFO +215 -0
- slurmscope-1.2.0/src/slurmscope.egg-info/SOURCES.txt +14 -0
- slurmscope-1.2.0/src/slurmscope.egg-info/dependency_links.txt +1 -0
- slurmscope-1.2.0/src/slurmscope.egg-info/entry_points.txt +2 -0
- slurmscope-1.2.0/src/slurmscope.egg-info/requires.txt +3 -0
- slurmscope-1.2.0/src/slurmscope.egg-info/top_level.txt +1 -0
- slurmscope-1.2.0/tests/test_slurmscope.py +842 -0
slurmscope-1.2.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Fabio Bozzoli
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: slurmscope
|
|
3
|
+
Version: 1.2.0
|
|
4
|
+
Summary: Live terminal dashboard for Slurm clusters: queue, GPUs, QoS, nodes, memory and storage.
|
|
5
|
+
Author: Fabio Bozzoli
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/FabioBozzoli/slurmscope
|
|
8
|
+
Project-URL: Source, https://github.com/FabioBozzoli/slurmscope
|
|
9
|
+
Project-URL: Issues, https://github.com/FabioBozzoli/slurmscope/issues
|
|
10
|
+
Keywords: slurm,hpc,cluster,gpu,dashboard,tui,monitoring
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Environment :: Console :: Curses
|
|
13
|
+
Classifier: Intended Audience :: Science/Research
|
|
14
|
+
Classifier: Intended Audience :: System Administrators
|
|
15
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
18
|
+
Classifier: Topic :: System :: Clustering
|
|
19
|
+
Classifier: Topic :: System :: Monitoring
|
|
20
|
+
Requires-Python: >=3.8
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Provides-Extra: test
|
|
24
|
+
Requires-Dist: pytest; extra == "test"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# slurmscope
|
|
28
|
+
|
|
29
|
+
Interactive terminal dashboard for any Slurm cluster: Python >= 3.8, standard library only, no dependencies.
|
|
30
|
+
|
|
31
|
+
## Install
|
|
32
|
+
|
|
33
|
+
pipx install slurmscope # recommended: isolated, puts `slurmscope` on your PATH
|
|
34
|
+
pip install --user slurmscope
|
|
35
|
+
python -m slurmscope # equivalent to the `slurmscope` command
|
|
36
|
+
|
|
37
|
+
It needs the Slurm client commands (`squeue`, `scontrol`) in `PATH` on the machine where you run it,
|
|
38
|
+
typically a login node.
|
|
39
|
+
|
|
40
|
+
Version 1.2 adds an AImageLab-HPC profile based on all six **Using the Cluster** documentation pages.
|
|
41
|
+
Live Slurm configuration, `squota`, and `susage` remain the source of operational values.
|
|
42
|
+
|
|
43
|
+
slurmscope # live, refresh every 1 s
|
|
44
|
+
slurmscope -p <partition> # start filtered on one partition
|
|
45
|
+
slurmscope -u <user> # whose jobs are highlighted / listed in the overview (default: you)
|
|
46
|
+
slurmscope --page 4 # start on memory / storage
|
|
47
|
+
slurmscope --once # single overview snapshot (pipe-friendly); --no-color, --ascii, --no-mouse
|
|
48
|
+
slurmscope --once --page 4 --scan-usage # RAM, quota, and directory sizes
|
|
49
|
+
|
|
50
|
+
## Views (1 2 3 4 / Tab, or click the tab bar)
|
|
51
|
+
- **Overview** - KPIs + history sparklines, per-partition load, GPU models (free GPUs), your jobs
|
|
52
|
+
(approximate priority rank, wait, backfill start estimate), pending reasons, top users, and a QoS summary.
|
|
53
|
+
The overview adapts to the terminal height (side-by-side panels from 150 columns, lists trimmed to fit);
|
|
54
|
+
it scrolls only when the terminal is too small, and the footer then shows the current line range.
|
|
55
|
+
Press **Q** or **Enter** to browse **all configured QoS**, including ones with no jobs.
|
|
56
|
+
Each row shows running/pending experiments, distinct users, allocated GPUs/CPUs, and
|
|
57
|
+
(on wider terminals) priority and maximum wall time. Search by name, description, or user;
|
|
58
|
+
sort or scroll through the complete list. **i** shows resource/job limits, policy flags,
|
|
59
|
+
user activity, and pending reasons. **Enter** on a QoS opens its experiments across all
|
|
60
|
+
visible users, clearing job-specific user/state/search filters; **Esc** returns to the browser.
|
|
61
|
+
The partition filter still applies to experiment counts/lists; **P** cycles back to all partitions.
|
|
62
|
+
QoS details distinguish shared GPU caps from individual caps and calculate visible GPU headroom
|
|
63
|
+
across all partitions. Partition policy adds wall-time bounds and billing weights; associations,
|
|
64
|
+
partition QoS, access rules and flags can further change the effective limits.
|
|
65
|
+
- **Jobs** - every job in the queue, scrollable. Running jobs of everybody or only yours.
|
|
66
|
+
QoS is searchable, sortable, shown in job details and, on wide terminals, in its own column.
|
|
67
|
+
Project accounts are searchable/sortable, shown on very wide terminals, and filterable with
|
|
68
|
+
**a** or `-A PROJECT`. **v** opens live `sprio` factors and the job's `scontrol` SiteFactor.
|
|
69
|
+
A displayed priority rank is not a guaranteed execution order or start-time estimate.
|
|
70
|
+
- **Nodes** - every node: state, CPU / GPU / memory bars, load, partitions.
|
|
71
|
+
**v** cycles minimum GPU VRAM; `--min-vram GB` sets it initially. Values come from live GPU
|
|
72
|
+
feature tags, not hardware assumptions or GPU-memory-utilization probes. Exact feature tokens
|
|
73
|
+
are searchable; node detail includes a quoted `--constraint` suggestion and raw GRES/tmpfs data.
|
|
74
|
+
- **Memory** - cluster configured/allocated RAM, capacity on nodes that are up and not drained,
|
|
75
|
+
unallocated RAM on those nodes, OS-reported free RAM, allocations by user (selected user first),
|
|
76
|
+
and RAM by node. Also includes the selected user's disk quota and on-demand directory sizes. The page is compact: users/nodes are packed into columns and trimmed to fit the terminal height;
|
|
77
|
+
it scrolls only when the terminal is too small.
|
|
78
|
+
|
|
79
|
+
## AImageLab-HPC integration
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
slurmscope --profile aimagelab
|
|
83
|
+
slurmscope --profile aimagelab --page 4
|
|
84
|
+
slurmscope --profile aimagelab --page 3 --min-vram 24 -p all_usr_prod
|
|
85
|
+
slurmscope --profile aimagelab --page 2 -A my_project
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
The default `--profile auto` recognizes an AImageLab/AILB cluster name, an `ailb-*` hostname,
|
|
89
|
+
or the presence of both `squota` and `susage`. `--profile generic` explicitly disables site integration.
|
|
90
|
+
You can force `aimagelab` when using a workstation with Slurm client tools.
|
|
91
|
+
|
|
92
|
+
- **Storage:** absolute `/homes/USER`, `/work`, and `/dres` paths; personal/project usage from
|
|
93
|
+
`squota`; capacity rows for associated project paths. Project counters include collaborators.
|
|
94
|
+
Recursive scans under `/work` and `/dres`, including aliases, are skipped in this profile.
|
|
95
|
+
- **B — Budgets:** all projects reported by `susage`, current-year remaining standard hours,
|
|
96
|
+
low/exhausted balances, and original counters. The overview also shows a compact summary.
|
|
97
|
+
Current-year columns are selected by their actual header; missing values stay unknown.
|
|
98
|
+
- **v on a job — Priority:** live component values, with the separate site factor. Recent-use
|
|
99
|
+
fairness changes priority; exhausted annual budgets can reject paid submissions. This follows
|
|
100
|
+
the newer Budget page where its annual-budget restriction qualifies the older Priority page.
|
|
101
|
+
- **E — Session:** loaded modules, local Python, job/session context and CUDA visibility.
|
|
102
|
+
On `ailb-login-02`, an active CUDA 12+ module produces the documented CUDA 11.8 compatibility hint.
|
|
103
|
+
No modules are loaded or unloaded, and another job's environment is never inferred.
|
|
104
|
+
- **D — Guide:** brief usage guidance, storage lifecycle, execution/transfer hints and links to
|
|
105
|
+
every source page. The reference was reviewed on 2026-11-01; live data takes precedence.
|
|
106
|
+
|
|
107
|
+
`squota` and `susage` run as the **logged-in Unix account**. `-u OTHER_USER` only changes job
|
|
108
|
+
highlighting and RAM summaries; it does not impersonate that user for these site commands.
|
|
109
|
+
Their identity is labelled in the UI. Missing commands, changed table formats and query errors
|
|
110
|
+
are shown explicitly; cached counters remain visible and marked stale after a failed refresh.
|
|
111
|
+
No job submissions, module changes, transfers or filesystem writes are performed by this integration.
|
|
112
|
+
|
|
113
|
+
| Documentation page | Integration |
|
|
114
|
+
|---|---|
|
|
115
|
+
| [Environment and Customization](https://ailb-web.ing.unimore.it/coldfront/documentation/LrU3q) | Session/module inspection and CUDA compatibility hint |
|
|
116
|
+
| [Scheduler and Job Submission](https://ailb-web.ing.unimore.it/coldfront/documentation/UKN8n) | Project accounts, QoS caps and partition policy |
|
|
117
|
+
| [Node Features and Constraints](https://ailb-web.ing.unimore.it/coldfront/documentation/flOS4) | Minimum VRAM filters and constraint tokens |
|
|
118
|
+
| [Budget and Accounting](https://ailb-web.ing.unimore.it/coldfront/documentation/GSOa2) | Live project budgets and balance summaries |
|
|
119
|
+
| [Job Priority](https://ailb-web.ing.unimore.it/coldfront/documentation/cHTQV) | Priority components, SiteFactor and approximate-rank labels |
|
|
120
|
+
| [File Systems and Data Management](https://ailb-web.ing.unimore.it/coldfront/documentation/UJMLD) | Site quotas, project scope and BeeGFS scan guard |
|
|
121
|
+
|
|
122
|
+
### QoS visibility
|
|
123
|
+
|
|
124
|
+
QoS definitions come from a read-only `sacctmgr show qos` query, cached for 60 seconds.
|
|
125
|
+
The experiment list uses jobs currently visible to `squeue`, not completed-job history.
|
|
126
|
+
Pending array ranges count as one queue row, matching the existing Jobs view.
|
|
127
|
+
Site privacy settings may hide other users' jobs. Definitions describe the accounting
|
|
128
|
+
database's QoS; access to submit jobs under them can still depend on your associations
|
|
129
|
+
and partition policies. If `sacctmgr`/SlurmDBD is unavailable or access is denied, a warning
|
|
130
|
+
appears and QoS found in the queue remain browsable; cached definitions are retained on failure.
|
|
131
|
+
|
|
132
|
+
### Memory and disk scope
|
|
133
|
+
|
|
134
|
+
RAM figures are **Slurm allocations, not measured process RSS**. Per-user RAM comes from
|
|
135
|
+
allocated TRES for active jobs; pending requests are not counted. Missing allocation data
|
|
136
|
+
(including the legacy `squeue -o` fallback) is marked **unknown**, not zero. Cluster allocation
|
|
137
|
+
is read from nodes, so it can differ from the sum of visible jobs. OS free memory is informational,
|
|
138
|
+
can be stale, and is reported with the number of nodes providing a value. Nodes are counted
|
|
139
|
+
once even when they belong to several partitions.
|
|
140
|
+
|
|
141
|
+
Disk data describes filesystems **mounted on the host running slurmscope**. Slurm does not
|
|
142
|
+
provide a cluster-wide filesystem inventory. By default the selected user's home and the
|
|
143
|
+
working directory are reported. Add shared storage paths explicitly:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
slurmscope --page 4 --storage-path /scratch --storage-path /projects
|
|
147
|
+
slurmscope -u alice --page 4 --usage-path /scratch/alice --usage-path /projects/alice
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
`--storage-path` adds filesystem-capacity rows. `--usage-path` selects directory scan roots
|
|
151
|
+
(repeatable; replaces the default selected-user home). Filesystem **used** includes all users;
|
|
152
|
+
**available** excludes reserved blocks. Paths on the same filesystem can show the same capacity,
|
|
153
|
+
so those rows must not be added. The **P** filter affects RAM/jobs, not storage scope.
|
|
154
|
+
|
|
155
|
+
In the generic profile, user ownership and limits are shown using `quota -w -u USER` when available
|
|
156
|
+
(the standard block unit is KiB). Quota support/permissions depend on the site; some shared
|
|
157
|
+
filesystems require their own quota tools. If quota is unavailable, press **d** to measure your
|
|
158
|
+
home or configured directories. Scans use `du -skx`, stay on one filesystem, run in the background,
|
|
159
|
+
and time out after 180 seconds per path (a home on a network filesystem can take over 30 s); each path
|
|
160
|
+
shows `scanning Ns` / `queued` and its result as soon as it is ready. A directory footprint includes files of all owners in that
|
|
161
|
+
directory, and is not a complete ownership-based quota. Nested paths must not be summed.
|
|
162
|
+
Permission failures and partial scans are labelled; previous samples remain visible during a rescan.
|
|
163
|
+
Directory scans run only on demand, or with `--once --page 4 --scan-usage`.
|
|
164
|
+
|
|
165
|
+
## Keys
|
|
166
|
+
| key | action |
|
|
167
|
+
|---|---|
|
|
168
|
+
| `↑ ↓ j k`, `PgUp PgDn`, `g G`, wheel | move / page / top / bottom |
|
|
169
|
+
| `Enter`, double-click | details: job (resources, files, queue position, ...) or node (+ its jobs) |
|
|
170
|
+
| `Q`, `Enter` on overview | full QoS browser; `Q` / `Esc` returns to the overview |
|
|
171
|
+
| `Enter`, `i` in QoS browser | experiments of all visible users / QoS limits and user activity |
|
|
172
|
+
| `4`, `d`, `r` | memory/storage page / scan directories / refresh RAM and storage |
|
|
173
|
+
| `B`, `E`, `D` | project budgets / current session and modules / AImageLab guide |
|
|
174
|
+
| `a`, `v` in Jobs | selected project filter / priority components |
|
|
175
|
+
| `v` in Nodes | cycle minimum GPU VRAM filter |
|
|
176
|
+
| `l` | tail the job's stdout (follows; `↑` scrolls back, `End` follows again, `e` switches to stderr) |
|
|
177
|
+
| `/` | search (words are ANDed: `bob a100 pending`) |
|
|
178
|
+
| `m` `u` `s` `f` | only mine / only the selected user / cycle state / GPU jobs (nodes: free GPUs) |
|
|
179
|
+
| `o` `O`, click header | next sort column / reverse / sort by the clicked column |
|
|
180
|
+
| `P` | cycle partition filter (all views) |
|
|
181
|
+
| `c` | cancel the selected job - own jobs only, asks `y/N` |
|
|
182
|
+
| `x` | clear filters |
|
|
183
|
+
| `p`/`Space`, `+ -` | pause, refresh interval (0.5-60 s) |
|
|
184
|
+
| `?`, `q` | scrollable help, quit (closes the current screen first) |
|
|
185
|
+
|
|
186
|
+
Mouse capture hides terminal text selection; hold Shift to select, or use `--no-mouse`.
|
|
187
|
+
|
|
188
|
+
## Cost and data
|
|
189
|
+
Per refresh: one `squeue -O ...tres-alloc...` and one `scontrol show node` (plus `squeue --start`
|
|
190
|
+
every 15 s if you have pending jobs; `scontrol show job` only while a detail/log screen is open).
|
|
191
|
+
QoS definitions refresh at most once per 60 s in a background worker. While page 4 is active,
|
|
192
|
+
user quota also refresh at most once per 60 s in the background.
|
|
193
|
+
Partition policies are cached for 60 s (`scontrol show partition -o`). The AImageLab profile
|
|
194
|
+
also refreshes `susage` at most once per 60 s. A priority report queries only the selected job
|
|
195
|
+
on demand, caches the result for 60 s, and supports explicit **r** refresh.
|
|
196
|
+
Only one query/scan of each kind can be in flight. Optional failures never stop queue polling.
|
|
197
|
+
`--once --page 1` queries definitions once; `--once --page 4` queries storage once.
|
|
198
|
+
The AImageLab overview snapshot also queries its project budget once. Site quota/budget tables
|
|
199
|
+
retain the original output when their column layout cannot be recognized; optional tools are
|
|
200
|
+
not required for core queue/node monitoring.
|
|
201
|
+
GPU counts come from the *allocated TRES*, so GPUs assigned by site defaults are counted too
|
|
202
|
+
(falls back to `squeue -o %b` on Slurm versions without `tres-alloc`).
|
|
203
|
+
|
|
204
|
+
Implementation references: [squeue](https://slurm.schedmd.com/squeue.html),
|
|
205
|
+
[QoS fields](https://slurm.schedmd.com/sacctmgr.html#SECTION_LIST/SHOW-QOS-FORMAT-OPTIONS),
|
|
206
|
+
and [scontrol](https://slurm.schedmd.com/scontrol.html).
|
|
207
|
+
|
|
208
|
+
## Development
|
|
209
|
+
|
|
210
|
+
git clone https://github.com/FabioBozzoli/slurmscope && cd slurmscope
|
|
211
|
+
pip install -e ".[test]"
|
|
212
|
+
python -m pytest
|
|
213
|
+
|
|
214
|
+
Layout: `src/slurmscope/cli.py` (dashboard and CLI), `tests/test_slurmscope.py`.
|
|
215
|
+
Released under the MIT license (see `LICENSE`).
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
# slurmscope
|
|
2
|
+
|
|
3
|
+
Interactive terminal dashboard for any Slurm cluster: Python >= 3.8, standard library only, no dependencies.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
pipx install slurmscope # recommended: isolated, puts `slurmscope` on your PATH
|
|
8
|
+
pip install --user slurmscope
|
|
9
|
+
python -m slurmscope # equivalent to the `slurmscope` command
|
|
10
|
+
|
|
11
|
+
It needs the Slurm client commands (`squeue`, `scontrol`) in `PATH` on the machine where you run it,
|
|
12
|
+
typically a login node.
|
|
13
|
+
|
|
14
|
+
Version 1.2 adds an AImageLab-HPC profile based on all six **Using the Cluster** documentation pages.
|
|
15
|
+
Live Slurm configuration, `squota`, and `susage` remain the source of operational values.
|
|
16
|
+
|
|
17
|
+
slurmscope # live, refresh every 1 s
|
|
18
|
+
slurmscope -p <partition> # start filtered on one partition
|
|
19
|
+
slurmscope -u <user> # whose jobs are highlighted / listed in the overview (default: you)
|
|
20
|
+
slurmscope --page 4 # start on memory / storage
|
|
21
|
+
slurmscope --once # single overview snapshot (pipe-friendly); --no-color, --ascii, --no-mouse
|
|
22
|
+
slurmscope --once --page 4 --scan-usage # RAM, quota, and directory sizes
|
|
23
|
+
|
|
24
|
+
## Views (1 2 3 4 / Tab, or click the tab bar)
|
|
25
|
+
- **Overview** - KPIs + history sparklines, per-partition load, GPU models (free GPUs), your jobs
|
|
26
|
+
(approximate priority rank, wait, backfill start estimate), pending reasons, top users, and a QoS summary.
|
|
27
|
+
The overview adapts to the terminal height (side-by-side panels from 150 columns, lists trimmed to fit);
|
|
28
|
+
it scrolls only when the terminal is too small, and the footer then shows the current line range.
|
|
29
|
+
Press **Q** or **Enter** to browse **all configured QoS**, including ones with no jobs.
|
|
30
|
+
Each row shows running/pending experiments, distinct users, allocated GPUs/CPUs, and
|
|
31
|
+
(on wider terminals) priority and maximum wall time. Search by name, description, or user;
|
|
32
|
+
sort or scroll through the complete list. **i** shows resource/job limits, policy flags,
|
|
33
|
+
user activity, and pending reasons. **Enter** on a QoS opens its experiments across all
|
|
34
|
+
visible users, clearing job-specific user/state/search filters; **Esc** returns to the browser.
|
|
35
|
+
The partition filter still applies to experiment counts/lists; **P** cycles back to all partitions.
|
|
36
|
+
QoS details distinguish shared GPU caps from individual caps and calculate visible GPU headroom
|
|
37
|
+
across all partitions. Partition policy adds wall-time bounds and billing weights; associations,
|
|
38
|
+
partition QoS, access rules and flags can further change the effective limits.
|
|
39
|
+
- **Jobs** - every job in the queue, scrollable. Running jobs of everybody or only yours.
|
|
40
|
+
QoS is searchable, sortable, shown in job details and, on wide terminals, in its own column.
|
|
41
|
+
Project accounts are searchable/sortable, shown on very wide terminals, and filterable with
|
|
42
|
+
**a** or `-A PROJECT`. **v** opens live `sprio` factors and the job's `scontrol` SiteFactor.
|
|
43
|
+
A displayed priority rank is not a guaranteed execution order or start-time estimate.
|
|
44
|
+
- **Nodes** - every node: state, CPU / GPU / memory bars, load, partitions.
|
|
45
|
+
**v** cycles minimum GPU VRAM; `--min-vram GB` sets it initially. Values come from live GPU
|
|
46
|
+
feature tags, not hardware assumptions or GPU-memory-utilization probes. Exact feature tokens
|
|
47
|
+
are searchable; node detail includes a quoted `--constraint` suggestion and raw GRES/tmpfs data.
|
|
48
|
+
- **Memory** - cluster configured/allocated RAM, capacity on nodes that are up and not drained,
|
|
49
|
+
unallocated RAM on those nodes, OS-reported free RAM, allocations by user (selected user first),
|
|
50
|
+
and RAM by node. Also includes the selected user's disk quota and on-demand directory sizes. The page is compact: users/nodes are packed into columns and trimmed to fit the terminal height;
|
|
51
|
+
it scrolls only when the terminal is too small.
|
|
52
|
+
|
|
53
|
+
## AImageLab-HPC integration
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
slurmscope --profile aimagelab
|
|
57
|
+
slurmscope --profile aimagelab --page 4
|
|
58
|
+
slurmscope --profile aimagelab --page 3 --min-vram 24 -p all_usr_prod
|
|
59
|
+
slurmscope --profile aimagelab --page 2 -A my_project
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
The default `--profile auto` recognizes an AImageLab/AILB cluster name, an `ailb-*` hostname,
|
|
63
|
+
or the presence of both `squota` and `susage`. `--profile generic` explicitly disables site integration.
|
|
64
|
+
You can force `aimagelab` when using a workstation with Slurm client tools.
|
|
65
|
+
|
|
66
|
+
- **Storage:** absolute `/homes/USER`, `/work`, and `/dres` paths; personal/project usage from
|
|
67
|
+
`squota`; capacity rows for associated project paths. Project counters include collaborators.
|
|
68
|
+
Recursive scans under `/work` and `/dres`, including aliases, are skipped in this profile.
|
|
69
|
+
- **B — Budgets:** all projects reported by `susage`, current-year remaining standard hours,
|
|
70
|
+
low/exhausted balances, and original counters. The overview also shows a compact summary.
|
|
71
|
+
Current-year columns are selected by their actual header; missing values stay unknown.
|
|
72
|
+
- **v on a job — Priority:** live component values, with the separate site factor. Recent-use
|
|
73
|
+
fairness changes priority; exhausted annual budgets can reject paid submissions. This follows
|
|
74
|
+
the newer Budget page where its annual-budget restriction qualifies the older Priority page.
|
|
75
|
+
- **E — Session:** loaded modules, local Python, job/session context and CUDA visibility.
|
|
76
|
+
On `ailb-login-02`, an active CUDA 12+ module produces the documented CUDA 11.8 compatibility hint.
|
|
77
|
+
No modules are loaded or unloaded, and another job's environment is never inferred.
|
|
78
|
+
- **D — Guide:** brief usage guidance, storage lifecycle, execution/transfer hints and links to
|
|
79
|
+
every source page. The reference was reviewed on 2026-11-01; live data takes precedence.
|
|
80
|
+
|
|
81
|
+
`squota` and `susage` run as the **logged-in Unix account**. `-u OTHER_USER` only changes job
|
|
82
|
+
highlighting and RAM summaries; it does not impersonate that user for these site commands.
|
|
83
|
+
Their identity is labelled in the UI. Missing commands, changed table formats and query errors
|
|
84
|
+
are shown explicitly; cached counters remain visible and marked stale after a failed refresh.
|
|
85
|
+
No job submissions, module changes, transfers or filesystem writes are performed by this integration.
|
|
86
|
+
|
|
87
|
+
| Documentation page | Integration |
|
|
88
|
+
|---|---|
|
|
89
|
+
| [Environment and Customization](https://ailb-web.ing.unimore.it/coldfront/documentation/LrU3q) | Session/module inspection and CUDA compatibility hint |
|
|
90
|
+
| [Scheduler and Job Submission](https://ailb-web.ing.unimore.it/coldfront/documentation/UKN8n) | Project accounts, QoS caps and partition policy |
|
|
91
|
+
| [Node Features and Constraints](https://ailb-web.ing.unimore.it/coldfront/documentation/flOS4) | Minimum VRAM filters and constraint tokens |
|
|
92
|
+
| [Budget and Accounting](https://ailb-web.ing.unimore.it/coldfront/documentation/GSOa2) | Live project budgets and balance summaries |
|
|
93
|
+
| [Job Priority](https://ailb-web.ing.unimore.it/coldfront/documentation/cHTQV) | Priority components, SiteFactor and approximate-rank labels |
|
|
94
|
+
| [File Systems and Data Management](https://ailb-web.ing.unimore.it/coldfront/documentation/UJMLD) | Site quotas, project scope and BeeGFS scan guard |
|
|
95
|
+
|
|
96
|
+
### QoS visibility
|
|
97
|
+
|
|
98
|
+
QoS definitions come from a read-only `sacctmgr show qos` query, cached for 60 seconds.
|
|
99
|
+
The experiment list uses jobs currently visible to `squeue`, not completed-job history.
|
|
100
|
+
Pending array ranges count as one queue row, matching the existing Jobs view.
|
|
101
|
+
Site privacy settings may hide other users' jobs. Definitions describe the accounting
|
|
102
|
+
database's QoS; access to submit jobs under them can still depend on your associations
|
|
103
|
+
and partition policies. If `sacctmgr`/SlurmDBD is unavailable or access is denied, a warning
|
|
104
|
+
appears and QoS found in the queue remain browsable; cached definitions are retained on failure.
|
|
105
|
+
|
|
106
|
+
### Memory and disk scope
|
|
107
|
+
|
|
108
|
+
RAM figures are **Slurm allocations, not measured process RSS**. Per-user RAM comes from
|
|
109
|
+
allocated TRES for active jobs; pending requests are not counted. Missing allocation data
|
|
110
|
+
(including the legacy `squeue -o` fallback) is marked **unknown**, not zero. Cluster allocation
|
|
111
|
+
is read from nodes, so it can differ from the sum of visible jobs. OS free memory is informational,
|
|
112
|
+
can be stale, and is reported with the number of nodes providing a value. Nodes are counted
|
|
113
|
+
once even when they belong to several partitions.
|
|
114
|
+
|
|
115
|
+
Disk data describes filesystems **mounted on the host running slurmscope**. Slurm does not
|
|
116
|
+
provide a cluster-wide filesystem inventory. By default the selected user's home and the
|
|
117
|
+
working directory are reported. Add shared storage paths explicitly:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
slurmscope --page 4 --storage-path /scratch --storage-path /projects
|
|
121
|
+
slurmscope -u alice --page 4 --usage-path /scratch/alice --usage-path /projects/alice
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`--storage-path` adds filesystem-capacity rows. `--usage-path` selects directory scan roots
|
|
125
|
+
(repeatable; replaces the default selected-user home). Filesystem **used** includes all users;
|
|
126
|
+
**available** excludes reserved blocks. Paths on the same filesystem can show the same capacity,
|
|
127
|
+
so those rows must not be added. The **P** filter affects RAM/jobs, not storage scope.
|
|
128
|
+
|
|
129
|
+
In the generic profile, user ownership and limits are shown using `quota -w -u USER` when available
|
|
130
|
+
(the standard block unit is KiB). Quota support/permissions depend on the site; some shared
|
|
131
|
+
filesystems require their own quota tools. If quota is unavailable, press **d** to measure your
|
|
132
|
+
home or configured directories. Scans use `du -skx`, stay on one filesystem, run in the background,
|
|
133
|
+
and time out after 180 seconds per path (a home on a network filesystem can take over 30 s); each path
|
|
134
|
+
shows `scanning Ns` / `queued` and its result as soon as it is ready. A directory footprint includes files of all owners in that
|
|
135
|
+
directory, and is not a complete ownership-based quota. Nested paths must not be summed.
|
|
136
|
+
Permission failures and partial scans are labelled; previous samples remain visible during a rescan.
|
|
137
|
+
Directory scans run only on demand, or with `--once --page 4 --scan-usage`.
|
|
138
|
+
|
|
139
|
+
## Keys
|
|
140
|
+
| key | action |
|
|
141
|
+
|---|---|
|
|
142
|
+
| `↑ ↓ j k`, `PgUp PgDn`, `g G`, wheel | move / page / top / bottom |
|
|
143
|
+
| `Enter`, double-click | details: job (resources, files, queue position, ...) or node (+ its jobs) |
|
|
144
|
+
| `Q`, `Enter` on overview | full QoS browser; `Q` / `Esc` returns to the overview |
|
|
145
|
+
| `Enter`, `i` in QoS browser | experiments of all visible users / QoS limits and user activity |
|
|
146
|
+
| `4`, `d`, `r` | memory/storage page / scan directories / refresh RAM and storage |
|
|
147
|
+
| `B`, `E`, `D` | project budgets / current session and modules / AImageLab guide |
|
|
148
|
+
| `a`, `v` in Jobs | selected project filter / priority components |
|
|
149
|
+
| `v` in Nodes | cycle minimum GPU VRAM filter |
|
|
150
|
+
| `l` | tail the job's stdout (follows; `↑` scrolls back, `End` follows again, `e` switches to stderr) |
|
|
151
|
+
| `/` | search (words are ANDed: `bob a100 pending`) |
|
|
152
|
+
| `m` `u` `s` `f` | only mine / only the selected user / cycle state / GPU jobs (nodes: free GPUs) |
|
|
153
|
+
| `o` `O`, click header | next sort column / reverse / sort by the clicked column |
|
|
154
|
+
| `P` | cycle partition filter (all views) |
|
|
155
|
+
| `c` | cancel the selected job - own jobs only, asks `y/N` |
|
|
156
|
+
| `x` | clear filters |
|
|
157
|
+
| `p`/`Space`, `+ -` | pause, refresh interval (0.5-60 s) |
|
|
158
|
+
| `?`, `q` | scrollable help, quit (closes the current screen first) |
|
|
159
|
+
|
|
160
|
+
Mouse capture hides terminal text selection; hold Shift to select, or use `--no-mouse`.
|
|
161
|
+
|
|
162
|
+
## Cost and data
|
|
163
|
+
Per refresh: one `squeue -O ...tres-alloc...` and one `scontrol show node` (plus `squeue --start`
|
|
164
|
+
every 15 s if you have pending jobs; `scontrol show job` only while a detail/log screen is open).
|
|
165
|
+
QoS definitions refresh at most once per 60 s in a background worker. While page 4 is active,
|
|
166
|
+
user quota also refresh at most once per 60 s in the background.
|
|
167
|
+
Partition policies are cached for 60 s (`scontrol show partition -o`). The AImageLab profile
|
|
168
|
+
also refreshes `susage` at most once per 60 s. A priority report queries only the selected job
|
|
169
|
+
on demand, caches the result for 60 s, and supports explicit **r** refresh.
|
|
170
|
+
Only one query/scan of each kind can be in flight. Optional failures never stop queue polling.
|
|
171
|
+
`--once --page 1` queries definitions once; `--once --page 4` queries storage once.
|
|
172
|
+
The AImageLab overview snapshot also queries its project budget once. Site quota/budget tables
|
|
173
|
+
retain the original output when their column layout cannot be recognized; optional tools are
|
|
174
|
+
not required for core queue/node monitoring.
|
|
175
|
+
GPU counts come from the *allocated TRES*, so GPUs assigned by site defaults are counted too
|
|
176
|
+
(falls back to `squeue -o %b` on Slurm versions without `tres-alloc`).
|
|
177
|
+
|
|
178
|
+
Implementation references: [squeue](https://slurm.schedmd.com/squeue.html),
|
|
179
|
+
[QoS fields](https://slurm.schedmd.com/sacctmgr.html#SECTION_LIST/SHOW-QOS-FORMAT-OPTIONS),
|
|
180
|
+
and [scontrol](https://slurm.schedmd.com/scontrol.html).
|
|
181
|
+
|
|
182
|
+
## Development
|
|
183
|
+
|
|
184
|
+
git clone https://github.com/FabioBozzoli/slurmscope && cd slurmscope
|
|
185
|
+
pip install -e ".[test]"
|
|
186
|
+
python -m pytest
|
|
187
|
+
|
|
188
|
+
Layout: `src/slurmscope/cli.py` (dashboard and CLI), `tests/test_slurmscope.py`.
|
|
189
|
+
Released under the MIT license (see `LICENSE`).
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "slurmscope"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Live terminal dashboard for Slurm clusters: queue, GPUs, QoS, nodes, memory and storage."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{name = "Fabio Bozzoli"}]
|
|
14
|
+
keywords = ["slurm", "hpc", "cluster", "gpu", "dashboard", "tui", "monitoring"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Environment :: Console :: Curses",
|
|
18
|
+
"Intended Audience :: Science/Research",
|
|
19
|
+
"Intended Audience :: System Administrators",
|
|
20
|
+
"Operating System :: POSIX :: Linux",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Topic :: System :: Clustering",
|
|
24
|
+
"Topic :: System :: Monitoring",
|
|
25
|
+
]
|
|
26
|
+
dependencies = []
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
test = ["pytest"]
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/FabioBozzoli/slurmscope"
|
|
33
|
+
Source = "https://github.com/FabioBozzoli/slurmscope"
|
|
34
|
+
Issues = "https://github.com/FabioBozzoli/slurmscope/issues"
|
|
35
|
+
|
|
36
|
+
[project.scripts]
|
|
37
|
+
slurmscope = "slurmscope.cli:main"
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.packages.find]
|
|
40
|
+
where = ["src"]
|
|
41
|
+
|
|
42
|
+
[tool.setuptools.dynamic]
|
|
43
|
+
version = {attr = "slurmscope.__version__"}
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
testpaths = ["tests"]
|
|
47
|
+
pythonpath = ["src"]
|