hpcmon 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. hpcmon-0.1.0/CHANGELOG.md +14 -0
  2. hpcmon-0.1.0/LICENSE +21 -0
  3. hpcmon-0.1.0/MANIFEST.in +3 -0
  4. hpcmon-0.1.0/PKG-INFO +431 -0
  5. hpcmon-0.1.0/README.md +393 -0
  6. hpcmon-0.1.0/hpcmon/__init__.py +3 -0
  7. hpcmon-0.1.0/hpcmon/__main__.py +8 -0
  8. hpcmon-0.1.0/hpcmon/cli.py +253 -0
  9. hpcmon-0.1.0/hpcmon/clusters/__init__.py +46 -0
  10. hpcmon-0.1.0/hpcmon/clusters/base.py +271 -0
  11. hpcmon-0.1.0/hpcmon/clusters/jupiter.py +214 -0
  12. hpcmon-0.1.0/hpcmon/clusters/leonardo.py +186 -0
  13. hpcmon-0.1.0/hpcmon/clusters/lumi.py +249 -0
  14. hpcmon-0.1.0/hpcmon/clusters/meluxina.py +235 -0
  15. hpcmon-0.1.0/hpcmon/clusters/slurm.py +89 -0
  16. hpcmon-0.1.0/hpcmon/config.py +347 -0
  17. hpcmon-0.1.0/hpcmon/models.py +319 -0
  18. hpcmon-0.1.0/hpcmon/render.py +420 -0
  19. hpcmon-0.1.0/hpcmon/runner.py +287 -0
  20. hpcmon-0.1.0/hpcmon/units.py +185 -0
  21. hpcmon-0.1.0/hpcmon.egg-info/PKG-INFO +431 -0
  22. hpcmon-0.1.0/hpcmon.egg-info/SOURCES.txt +55 -0
  23. hpcmon-0.1.0/hpcmon.egg-info/dependency_links.txt +1 -0
  24. hpcmon-0.1.0/hpcmon.egg-info/entry_points.txt +2 -0
  25. hpcmon-0.1.0/hpcmon.egg-info/requires.txt +14 -0
  26. hpcmon-0.1.0/hpcmon.egg-info/top_level.txt +1 -0
  27. hpcmon-0.1.0/pyproject.toml +65 -0
  28. hpcmon-0.1.0/requirements.txt +11 -0
  29. hpcmon-0.1.0/setup.cfg +8 -0
  30. hpcmon-0.1.0/tests/__init__.py +0 -0
  31. hpcmon-0.1.0/tests/conftest.py +96 -0
  32. hpcmon-0.1.0/tests/fixtures/jupiter/cpuquota.json +23 -0
  33. hpcmon-0.1.0/tests/fixtures/jupiter/projectdata.json +47 -0
  34. hpcmon-0.1.0/tests/fixtures/jupiter/userdata.json +40 -0
  35. hpcmon-0.1.0/tests/fixtures/leonardo/cindata.txt +6 -0
  36. hpcmon-0.1.0/tests/fixtures/leonardo/saldo.txt +5 -0
  37. hpcmon-0.1.0/tests/fixtures/leonardo/saldo_dcgp.txt +1 -0
  38. hpcmon-0.1.0/tests/fixtures/lumi/allocations.txt +4 -0
  39. hpcmon-0.1.0/tests/fixtures/lumi/projectinfo.txt +24 -0
  40. hpcmon-0.1.0/tests/fixtures/lumi/workspaces.txt +32 -0
  41. hpcmon-0.1.0/tests/fixtures/meluxina/myquota.txt +13 -0
  42. hpcmon-0.1.0/tests/fixtures/meluxina/myquota_rate_limited.txt +1 -0
  43. hpcmon-0.1.0/tests/fixtures/slurm/squeue.txt +5 -0
  44. hpcmon-0.1.0/tests/test_cli.py +1889 -0
  45. hpcmon-0.1.0/tests/test_clusters_base.py +1283 -0
  46. hpcmon-0.1.0/tests/test_config.py +927 -0
  47. hpcmon-0.1.0/tests/test_gpu_hours.py +836 -0
  48. hpcmon-0.1.0/tests/test_jupiter.py +1281 -0
  49. hpcmon-0.1.0/tests/test_leonardo.py +794 -0
  50. hpcmon-0.1.0/tests/test_lumi.py +1157 -0
  51. hpcmon-0.1.0/tests/test_main.py +144 -0
  52. hpcmon-0.1.0/tests/test_meluxina.py +941 -0
  53. hpcmon-0.1.0/tests/test_models.py +822 -0
  54. hpcmon-0.1.0/tests/test_render.py +1749 -0
  55. hpcmon-0.1.0/tests/test_runner.py +1356 -0
  56. hpcmon-0.1.0/tests/test_units.py +673 -0
@@ -0,0 +1,14 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0 (2026-11-19)
4
+
5
+ First release.
6
+
7
+ - Shows, for every configured cluster, the compute budgets (remaining until the end of the project
8
+ and, where budgets are split per month, until the end of the month), the storage quotas and the
9
+ user's running and pending jobs.
10
+ - Supports LUMI, JUPITER, Leonardo and MeluXina, plus the jobs of any other Slurm cluster.
11
+ - Shows the GPUs of the nodes of every cluster and estimates every budget that can be spent on
12
+ them in GPU-hours, with a summary of the GPU-hours left on all clusters.
13
+ - One SSH session per cluster, all clusters queried in parallel, with a time limit per cluster.
14
+ - TOML configuration (`~/.config/hpcmon/config.toml`) and JSON output (`--json`).
hpcmon-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Dimitrios Karageorgiou
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,3 @@
1
+ include README.md CHANGELOG.md LICENSE requirements.txt
2
+ recursive-include tests *.py *.txt *.json
3
+ global-exclude __pycache__ *.py[cod]
hpcmon-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,431 @@
1
+ Metadata-Version: 2.4
2
+ Name: hpcmon
3
+ Version: 0.1.0
4
+ Summary: Monitor compute budgets, storage quotas and jobs on EuroHPC clusters (LUMI, JUPITER, Leonardo, MeluXina) from your terminal.
5
+ Author: Dimitrios Karageorgiou
6
+ License-Expression: MIT
7
+ Keywords: hpc,eurohpc,slurm,lumi,jupiter,leonardo,meluxina,allocation,quota,monitoring
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Environment :: Console
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: Intended Audience :: System Administrators
12
+ Classifier: Operating System :: POSIX
13
+ Classifier: Operating System :: MacOS
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3 :: Only
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Programming Language :: Python :: 3.14
22
+ Classifier: Topic :: System :: Clustering
23
+ Classifier: Topic :: System :: Monitoring
24
+ Requires-Python: >=3.9
25
+ Description-Content-Type: text/markdown
26
+ License-File: LICENSE
27
+ Requires-Dist: rich>=12.0
28
+ Requires-Dist: tomli>=1.1; python_version < "3.11"
29
+ Provides-Extra: test
30
+ Requires-Dist: pytest>=7; extra == "test"
31
+ Requires-Dist: pytest-cov>=4; extra == "test"
32
+ Provides-Extra: dev
33
+ Requires-Dist: pytest>=7; extra == "dev"
34
+ Requires-Dist: pytest-cov>=4; extra == "dev"
35
+ Requires-Dist: build; extra == "dev"
36
+ Requires-Dist: twine; extra == "dev"
37
+ Dynamic: license-file
38
+
39
+ # hpcmon
40
+
41
+ `hpcmon` is a command-line tool that shows, in one view, the resources you have left on the
42
+ EuroHPC clusters you have access to:
43
+
44
+ - **compute budgets**: what remains until the end of each project, and until the end of the
45
+ month for clusters that split budgets per month;
46
+ - **storage**: used and available space (and file counts) of your home and project areas;
47
+ - **jobs**: your running and pending Slurm jobs;
48
+ - **GPUs**: the GPUs of the nodes of every cluster, and every budget that can be spent on them
49
+ converted to an estimate in **GPU-hours**, so that the budgets of different centres (core-hours,
50
+ node-hours, ...) can be compared.
51
+
52
+ It connects to every cluster with your own `ssh` command, runs the site's accounting tools
53
+ (`lumi-allocations`, `jutil`, `saldo`, `cindata`, `myquota`, `squeue`, ...) and presents the
54
+ results in the same format for every cluster.
55
+
56
+ Supported systems: **LUMI**, **JUPITER**, **Leonardo** and **MeluXina**. Any other Slurm cluster
57
+ can be added to show its jobs only.
58
+
59
+ ## Installation
60
+
61
+ ```bash
62
+ pip install hpcmon
63
+ ```
64
+
65
+ or, from a clone of the repository:
66
+
67
+ ```bash
68
+ pip install .
69
+ ```
70
+
71
+ `hpcmon` requires Python 3.9 or later on your local machine (Linux or macOS; the automated tests
72
+ run on Linux). Nothing needs to be installed on the clusters.
73
+
74
+ ## Prerequisites: SSH access
75
+
76
+ `hpcmon` runs locally and connects to each cluster through SSH. By default it runs
77
+ `ssh lumi`, `ssh jupiter`, `ssh leonardo` and `ssh meluxina`, so the simplest setup is a host alias
78
+ for every cluster in `~/.ssh/config`:
79
+
80
+ ```sshconfig
81
+ Host lumi
82
+ HostName lumi.csc.fi
83
+ User <your LUMI username>
84
+ IdentityFile ~/.ssh/id_lumi
85
+
86
+ Host leonardo
87
+ HostName login.leonardo.cineca.it
88
+ User <your CINECA username>
89
+ ```
90
+
91
+ If you use other host names or need extra SSH options, set them in the configuration file (see
92
+ *Configuration* below).
93
+
94
+ Several centres require multi-factor authentication or short-lived certificates (e.g. JUPITER's
95
+ OTP, or Leonardo's `step` certificates). Make sure that `ssh <cluster>` works without prompting
96
+ before you run `hpcmon`. An SSH *ControlMaster* is the easiest way to do this: you authenticate
97
+ once, and the connection is reused for hours:
98
+
99
+ ```sshconfig
100
+ Host lumi jupiter leonardo meluxina
101
+ ControlMaster auto
102
+ ControlPath ~/.ssh/control-%h-%p-%r
103
+ ControlPersist 12h
104
+ ```
105
+
106
+ If a cluster does prompt for a password or OTP, run `hpcmon --sequential` (or set
107
+ `parallel = false`). The clusters are then queried one after the other, without the progress
108
+ spinner, so the SSH prompts are visible and don't overlap. If you would rather have `hpcmon` fail
109
+ fast than prompt, add `-o BatchMode=yes` to the SSH command of that cluster.
110
+
111
+ ## Usage
112
+
113
+ ```bash
114
+ hpcmon # all enabled clusters, all sections
115
+ hpcmon lumi leonardo # only these clusters
116
+ hpcmon --jobs # only the jobs (fast)
117
+ hpcmon leonardo --compute # options and cluster names can be mixed freely
118
+ hpcmon --json | jq . # machine-readable output
119
+ ```
120
+
121
+ Example output (illustrative data):
122
+
123
+ ```text
124
+ LUMI · GPU nodes: 4× AMD MI250X 128 GB (2 GCDs each) ─────────────────────────────────────────────────────────
125
+ Compute allocations
126
+
127
+ Project Resource Total Remaining Ends (days left) Notes
128
+ ─────────────────────────────────────────────────────────────────────────────────────────────────────
129
+ project_465000123 CPU [core-h] 50,000 50,000 (100%) 2027-04-04 (137d)
130
+ project_465000123 GPU [GPU-h] 12,000 12,000 (100%) 2027-04-04 (137d)
131
+ project_465000123 Storage [TB-h] 200,000 200,000 (100%) 2027-04-04 (137d) storage billing
132
+
133
+ Storage
134
+
135
+ Area Used Quota Free Use Files / quota
136
+ ──────────────────────────────────────────────────────────────────────────────────────
137
+ /users/jdoe1234 20.0 KiB 25.0 GiB 25.0 GiB 0.0% 5 / 150k
138
+ /project/project_465000123 4.0 KiB 100.0 GiB 100.0 GiB 0.0% 1 / 200k
139
+ /scratch/project_465000123 4.0 KiB 20.0 TiB 20.0 TiB 0.0% 1 / 4M
140
+ /flash/project_465000123 4.0 KiB 1.0 TiB 1.0 TiB 0.0% 1 / 3M
141
+
142
+ Jobs: 2 running, 1 pending
143
+
144
+ Job ID Name Partition State Time / limit Nodes Node list / reason
145
+ ──────────────────────────────────────────────────────────────────────────────────────────────────────────
146
+ 1234567 train|model [v2] standard-g RUNNING 1:02:03 / 2-00:00:00 4 nid[005000-005003]
147
+ 1234570 cleanup dev-g COMPLETING 10:00 / 30:00 1 nid005100
148
+ 1234568 eval small-g PENDING 0:00 / 1:00:00 1 (Priority)
149
+
150
+
151
+ Leonardo · GPU nodes: 4× NVIDIA A100 64 GB ───────────────────────────────────────────────────────────────────
152
+ Compute allocations
153
+
154
+ Month Month
155
+ Project Resource Total Remaining total remaining Ends (days left)
156
+ ──────────────────────────────────────────────────────────────────────────────────────────────────────
157
+ EUHPC_D00_001 Booster [core-h] 96,000 92,813 (97%) 8,250 8,250 (100%) 2027-06-30 (224d)
158
+ ~11,602 GPU-h ~1,031 GPU-h
159
+
160
+ Storage
161
+
162
+ Area Used Quota Free Use Files / quota
163
+ ───────────────────────────────────────────────────────────────────────────────────────────────────────
164
+ /leonardo_work/EUHPC_D00_001 87.0 GiB 2.0 TiB 1.9 TiB 4.2% -
165
+ /leonardo_scratch/fast/EUHPC_D00_001 4.0 KiB 2.0 TiB 2.0 TiB 0.0% -
166
+ /leonardo_scratch/large/userexternal/jdoe5678 4.0 KiB no quota - - -
167
+ /leonardo/pub/userexternal/jdoe5678 4.0 KiB 60.0 GiB 60.0 GiB 0.0% -
168
+ /leonardo/home/userexternal/jdoe5678 3.4 GiB 60.0 GiB 56.6 GiB 5.7% -
169
+
170
+ Note: as of 3 hours ago
171
+ No jobs in the queue.
172
+
173
+ MeluXina · GPU nodes: 4× NVIDIA A100 40 GB ───────────────────────────────────────────────────────────────────
174
+ Compute allocations
175
+
176
+ Month Month
177
+ Project Resource Total Remaining total remaining Ends (days left)
178
+ ────────────────────────────────────────────────────────────────────────────────────
179
+ p200001 GPU [node-h] - - 210 183 (87%) -
180
+ ~732 GPU-h
181
+
182
+ Note: current month only
183
+ ...
184
+
185
+ All clusters ─────────────────────────────────────────────────────────────────────────────────────────────────
186
+ GPU-hours left (estimated)
187
+
188
+ Month
189
+ Cluster Project Remaining remaining Ends (days left)
190
+ ───────────────────────────────────────────────────────────────────────────
191
+ LUMI project_465000123 12,000 (100%) - 2027-04-04 (137d)
192
+ Leonardo EUHPC_D00_001 11,602 (97%) 1,031 (100%) 2027-06-30 (224d)
193
+ MeluXina p200001 - 732 (87%) -
194
+
195
+ Note: GPU-hours (of the GPUs named next to each cluster) are estimates: they
196
+ assume that jobs use all the GPUs of the nodes they are charged for (on
197
+ shared nodes, no more than the cores and memory that come with their GPUs).
198
+ ```
199
+
200
+ *Remaining* is what is left until the end of the project. *Month remaining* is what is left until
201
+ the end of the current month, for clusters that split budgets per month. Below the amounts of the
202
+ budgets that can be spent on GPU nodes, `~N GPU-h` is the estimate of what they buy in GPU-hours
203
+ (see *GPUs and GPU-hours* below). When several clusters have such budgets, a summary of the
204
+ GPU-hours left on all of them is printed last. Remaining amounts are
205
+ green, yellow or red when more than 50%, 20–50% or less than 20% is left, and dimmed once the
206
+ project has ended. End dates turn yellow 30 days and red 14 days before the end. Storage usage
207
+ turns yellow above 80% and red above 95% of the quota.
208
+
209
+ The tables adapt to the width of the terminal: on narrow terminals long values wrap over several
210
+ lines instead of being truncated.
211
+
212
+ ### Options
213
+
214
+ | Option | Description |
215
+ | --- | --- |
216
+ | `CLUSTER ...` | Names of the clusters to query (default: all enabled clusters). A disabled cluster is queried when it is named explicitly. |
217
+ | `--compute`, `--storage`, `--jobs` | Show only these sections (any combination). By default all sections are shown. |
218
+ | `--json` | Print the results as JSON instead of tables. |
219
+ | `-c PATH`, `--config PATH` | Use this configuration file. |
220
+ | `-t SECONDS`, `--timeout SECONDS` | Time limit for each cluster (overrides the configuration). |
221
+ | `--sequential` | Query the clusters one after the other (and without the progress spinner). |
222
+ | `--no-color` | Disable colours. |
223
+ | `--debug` | Log the commands run on the clusters and their raw output (useful to report parsing problems). |
224
+ | `--init-config [--force]` | Write a commented default configuration file and exit (`--force` overwrites an existing file). |
225
+ | `--show-config` | Show the configured clusters and exit. |
226
+ | `-V`, `--version` | Show the version and exit. |
227
+
228
+ Exit status:
229
+
230
+ - `0`: every cluster was queried successfully. Warnings don't count.
231
+ - `1`: the data of at least one cluster is incomplete. The problem is shown under that cluster.
232
+ - `2`: usage or configuration error.
233
+ - `130`: interrupted with Ctrl-C.
234
+
235
+ ## What is shown for each cluster
236
+
237
+ | Cluster | Compute budgets | Per month | Project end | Storage | Commands used |
238
+ | --- | --- | --- | --- | --- | --- |
239
+ | LUMI | CPU core-hours, GPU-hours and storage TB-hours per project | no (LUMI does not split budgets per month) | yes ("days of compute left") | home, `/project`, `/scratch`, `/flash` (space and files) | `lumi-allocations`, `lumi-ldap-projectinfo`, `lumi-workspaces` |
240
+ | JUPITER | core-hours per budget account on the system | for budgets that JSC does not account in "fixed" mode, as far as `jutil` reports them | yes (end of the accounting period) | home, project, scratch and fast-scratch directories (space and inodes) | `jutil user cpuquota`, `jutil user dataquota`, `jutil project dataquota` |
241
+ | Leonardo | local core-hours per account, Booster and DCGP partitions | yes (`monthTotal`/`monthConsumed`) | yes | home, public, work and scratch areas | `saldo -b`, `saldo -b --dcgp`, `cindata` |
242
+ | MeluXina | node-hours of the current month (CPU, GPU, FPGA, large memory) | yes: this is all that MeluXina reports | no (it can be set in the configuration) | home and project directories (space and files) | `myquota` |
243
+ | any Slurm cluster (`type = "slurm"`) | – | – | – | – | – |
244
+
245
+ Jobs are listed with `squeue` on all clusters.
246
+
247
+ Notes and limitations:
248
+
249
+ - Amounts are shown in each centre's own unit (core-hours, GPU-hours, node-hours, ...). The budgets
250
+ that can be spent on GPU nodes are also estimated in GPU-hours (see below).
251
+ - **LUMI**: the storage billing budget (TB-hours) is listed with the compute budgets, as LUMI accounts
252
+ for it the same way. It is marked "storage billing". The end of a project is computed from the
253
+ "days of compute left" reported by LUMI.
254
+ - **JUPITER**: budgets are reported for the current system only (`jutil ... --currsys`). *Remaining*
255
+ is the quota minus the consumption until the end of the accounting period. For accounting modes
256
+ other than "fixed", the accounting mode and the amount available right now (when it differs) are
257
+ shown as notes. Only the "fixed" mode could be checked on a real system. If you are not allowed to
258
+ read the quotas of a project, your own usage in it is shown instead.
259
+ - **Leonardo**: `cindata` takes around 30–40 seconds, so the storage section of Leonardo is the
260
+ slowest part of a full query. Use `hpcmon leonardo --compute --jobs` for a quick look. For project
261
+ areas (work, fast scratch), the usage and size of the whole area are shown. For personal areas
262
+ (paths under `userexternal`/`userinternal`), your own usage and quota are shown. `cindata`
263
+ refreshes its figures periodically, so their age is shown as a note.
264
+ - **MeluXina**: `myquota` reports neither the total budget of a project nor its end date, so only
265
+ the current month is shown. You can set the end dates of your projects with `project_end` in the
266
+ configuration. `myquota` also refuses to run twice within 10 seconds. If you run `hpcmon` again too
267
+ quickly, MeluXina asks you to retry later.
268
+ - Storage quotas show the *soft* limit, i.e. the quota you are expected to stay below. "no quota"
269
+ means that the cluster reports no limit for that area.
270
+
271
+ ## GPUs and GPU-hours
272
+
273
+ Each cluster's name is followed by the GPUs of its GPU nodes. The budgets that can be spent on these
274
+ nodes are converted to **GPU-hours**: hours of one of these GPUs. GPUs are taken to be the primary
275
+ resource, so the estimates assume that jobs use the GPU nodes as efficiently as the accounting
276
+ allows:
277
+
278
+ - on clusters that allocate **whole nodes** (JUPITER, MeluXina, LUMI's `standard-g`), jobs use all
279
+ the GPUs of their nodes. A job that uses 1 GPU of a 4-GPU node still pays for 4 GPU-hours per hour;
280
+ - on clusters that **share nodes** (Leonardo's Booster, LUMI's `small-g` and `dev-g`), jobs request
281
+ no more than the cores and memory that come with their GPUs (e.g. 25% of the node for 1 of its 4
282
+ GPUs). Asking for more than the cores or memory billed with a GPU (last column) makes the same
283
+ GPU-hour cost more.
284
+
285
+ | Cluster | GPU nodes | Budget converted | GPU-hours per unit | Allocation | Billed with each GPU |
286
+ | --- | --- | --- | --- | --- | --- |
287
+ | LUMI | 4× AMD MI250X 128 GB, 2 GCDs each (Slurm counts 8 GPUs of 64 GB) | GPU [GPU-h] | 1 (a LUMI GPU-hour is one MI250X, i.e. 2 GCDs, for an hour) | shared (`small-g`, `dev-g`) or whole nodes (`standard-g`) | 16 cores and 128 GB per MI250X: LUMI bills each GCD half a GPU-hour per hour, or more if a job takes more than 8 cores or 64 GB per GCD (only 56 cores of a node, 7 per GCD, are usable) |
288
+ | JUPITER | 4× NVIDIA GH200 96 GB (288 Grace cores per node) | jupiter_booster [core-h] | 1/72 (a node-hour is charged 288 core-hours) | whole nodes | – |
289
+ | Leonardo | 4× NVIDIA A100 64 GB (32 cores per node) | Booster [core-h] | 1/8 (a node-hour is charged 32 core-hours) | shared | 8 cores and 1/4 of the node memory (the default memory follows the cores) |
290
+ | MeluXina | 4× NVIDIA A100 40 GB | GPU [node-h] | 4 | whole nodes | – |
291
+
292
+ The other budgets cannot be spent on GPU nodes and get no estimate: LUMI's CPU core-hours (LUMI-C
293
+ and the visualisation nodes) and storage TB-hours, Leonardo's DCGP core-hours (CPU-only nodes), and
294
+ MeluXina's CPU, FPGA and large-memory node-hours.
295
+
296
+ The rates come from the centres' accounting rules, and were checked against the accounting of real
297
+ jobs: the core-hours charged by JUPITER and Leonardo, and the node-hours charged by MeluXina, are the
298
+ node-hours of the jobs times 288, 32 and 1. GPUs of different clusters are of course not equally
299
+ fast: the GPU-hours tell how long you can use the GPUs of each cluster, not how much work they do.
300
+
301
+ The rates can be changed, or added for other budgets of the supported systems, with the
302
+ `gpu_hours_per_unit` setting, and the GPU description with `gpus` (see *Configuration*). For generic
303
+ Slurm clusters (`type = "slurm"`), which show jobs only, `gpus` describes their GPU nodes. The estimates are also in the JSON output
304
+ (`gpu_hours_per_unit`, `gpu_hours_total`, `gpu_hours_remaining`, `month_gpu_hours_total`,
305
+ `month_gpu_hours_remaining` of every budget, and `gpus` of every cluster).
306
+
307
+ ## Configuration
308
+
309
+ `hpcmon` works without any configuration and queries the four clusters with `ssh <name>`. To change
310
+ this, create a configuration file:
311
+
312
+ ```bash
313
+ hpcmon --init-config # writes ~/.config/hpcmon/config.toml
314
+ hpcmon --show-config # shows what hpcmon will use
315
+ ```
316
+
317
+ The configuration is read from the file given with `--config`, or else from the file named by the
318
+ `HPCMON_CONFIG` environment variable. With either of these, the file must exist. Otherwise it is
319
+ read from `$XDG_CONFIG_HOME/hpcmon/config.toml` (by default `~/.config/hpcmon/config.toml`), if that
320
+ file exists. It is a [TOML](https://toml.io) file:
321
+
322
+ ```toml
323
+ [settings]
324
+ timeout = 180 # time limit for each cluster, in seconds
325
+ parallel = true # query all clusters at the same time
326
+ login_shell = true # run remote commands in a login shell (loads the site's PATH)
327
+
328
+ [clusters.lumi]
329
+ type = "lumi" # lumi, jupiter, leonardo, meluxina or slurm
330
+ ssh = "ssh lumi" # how to connect: a string or a list of arguments
331
+ projects = ["project_465000123"] # only show these projects (default: all)
332
+
333
+ [clusters.jupiter]
334
+ ssh = ["ssh", "-o", "ConnectTimeout=20", "jupiter"] # `type` can be omitted when the name is a type
335
+
336
+ [clusters.leonardo]
337
+ ssh = "ssh -l jdoe0000 login.leonardo.cineca.it"
338
+ timeout = 240 # per-cluster override
339
+
340
+ [clusters.meluxina]
341
+ ssh = "ssh meluxina"
342
+ enabled = false # skipped unless named on the command line: `hpcmon meluxina`
343
+
344
+ [clusters.meluxina.project_end] # end dates for clusters that don't report them
345
+ p200001 = 2027-06-30
346
+
347
+ [clusters.my-uni-cluster] # any other Slurm cluster: shows the jobs only
348
+ type = "slurm"
349
+ ssh = "ssh hpc.example.org"
350
+ gpus = "8× NVIDIA H100 80 GB" # describes its GPU nodes
351
+ ```
352
+
353
+ When the file has a `[clusters]` section, only the clusters listed in it are used. A cluster table
354
+ accepts these keys:
355
+
356
+ | Key | Default | Description |
357
+ | --- | --- | --- |
358
+ | `type` | the table name | The kind of system, selecting how it is queried: `lumi`, `jupiter`, `leonardo`, `meluxina` or `slurm`. |
359
+ | `ssh` | `"ssh <name>"` | The command used to connect. `hpcmon` appends `bash -l -s` to it and sends the commands on standard input. |
360
+ | `enabled` | `true` | Whether the cluster is queried when no cluster is named on the command line. |
361
+ | `timeout` | `settings.timeout` (180) | Time limit, in seconds, for the commands run on this cluster. Another 15 seconds are allowed for connecting. |
362
+ | `login_shell` | `settings.login_shell` (true) | Whether to run the commands in a login shell. |
363
+ | `projects` | all projects | Only show these projects (case-insensitive): their compute budgets, their storage areas and the jobs charged to them. Personal storage is always shown. A warning tells how many jobs are hidden, and names any configured project the cluster does not know (to catch typos). |
364
+ | `project_end` | – | A table of `project = date` giving project end dates, used when the cluster does not report them. |
365
+ | `gpus` | built in for the supported systems | Description of the GPUs of a GPU node, shown next to the cluster name. `""` hides it. |
366
+ | `gpu_hours_per_unit` | built in (see *GPUs and GPU-hours*) | A table of `resource = rate`: the GPU-hours bought by one unit of the budgets of that resource, replacing the built-in rate. The resource is the *Resource* column without the `[unit]` (e.g. `GPU`, `Booster` or `jupiter_booster`; case-insensitive). `0` disables the estimate. A resource that matches no budget of the cluster is reported as a warning. Example: `[clusters.leonardo.gpu_hours_per_unit]` with `Booster = 0.125`. |
367
+
368
+ You can use any SSH setup: jump hosts (`ssh -J gateway lumi`), a different user
369
+ (`ssh -l user host`), or a wrapper script. The command only has to run its last argument
370
+ (`bash -l -s`) on the cluster with the standard input attached.
371
+
372
+ ## How it works
373
+
374
+ For every cluster, `hpcmon` opens **one** SSH session. In it, a small bash script runs all the
375
+ commands needed for the requested sections, one after the other. The fast ones (jobs, compute) go
376
+ first, and the slow storage tools go last. Each command's output is enclosed in markers, so any
377
+ banner printed by the login shell is ignored, and a failing command affects only its own section.
378
+ The commands share the time limit of the cluster: a command that exceeds what is left of it is
379
+ stopped, and the results of the commands that completed are still shown. Each command's standard
380
+ input comes from `/dev/null`. All clusters are queried in parallel, and the results are printed in
381
+ the order of the configuration as soon as they are available.
382
+
383
+ Each supported system has a class in `hpcmon/clusters/` that derives from `hpcmon.clusters.Cluster`
384
+ (or `SlurmCluster`). The class declares the remote commands needed for each section, and parses
385
+ their output into the common model of `hpcmon/models.py`: `ComputeBudget`, `StorageQuota` and `Job`.
386
+ To support another cluster:
387
+
388
+ 1. Create `hpcmon/clusters/<name>.py` with a subclass of `SlurmCluster`. Set `type_name` and
389
+ `display_name`, and implement `compute_commands()`, `storage_commands()`, `parse_compute()` and
390
+ `parse_storage()`. Set `gpus` (the GPUs of a node) and `gpu_rates` (GPU-hours per unit of each
391
+ `(resource, unit)` of budget that can be spent on GPUs).
392
+ 2. Add the class to `CLUSTER_TYPES` in `hpcmon/clusters/__init__.py`.
393
+ 3. Add the real output of the tools to `tests/fixtures/<name>/` (anonymised) and tests in
394
+ `tests/test_<name>.py`.
395
+
396
+ `hpcmon --debug` prints the raw output of every remote command, which is the easiest way to capture
397
+ fixtures.
398
+
399
+ ## Development
400
+
401
+ From a clone of the repository:
402
+
403
+ ```bash
404
+ pip install -r requirements.txt
405
+ pip install -e .
406
+
407
+ pytest # runs the test suite with coverage (it requires 100%)
408
+ ```
409
+
410
+ The tests never connect to a cluster. They use mock outputs of the cluster tools, stored in
411
+ `tests/fixtures/`.
412
+
413
+ The tests also run on GitHub Actions (`.github/workflows/tests.yml`) for every push and pull
414
+ request, on all the supported Python versions and with the oldest supported dependencies.
415
+
416
+ ## PyPI Releases
417
+
418
+ Releases are published on PyPI by GitHub Actions (`.github/workflows/release.yml`) when a release
419
+ is published on GitHub. The workflow runs the tests, checks that the release tag is the package
420
+ version, builds the sdist and the wheel, and uploads them with PyPI's *trusted publishing*, so no
421
+ API token is stored anywhere.
422
+
423
+ To release a version:
424
+
425
+ 1. Set `__version__` in `hpcmon/__init__.py` (e.g. `"0.2.0"`) and add the changes to `CHANGELOG.md`.
426
+ 2. Commit and push.
427
+ 3. Create a release on GitHub with a new tag named after the version (`v0.2.0` or `0.2.0`).
428
+
429
+ ## License
430
+
431
+ MIT. See [LICENSE](LICENSE).