hpcmon 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hpcmon/__init__.py +3 -0
- hpcmon/__main__.py +8 -0
- hpcmon/cli.py +253 -0
- hpcmon/clusters/__init__.py +46 -0
- hpcmon/clusters/base.py +271 -0
- hpcmon/clusters/jupiter.py +214 -0
- hpcmon/clusters/leonardo.py +186 -0
- hpcmon/clusters/lumi.py +249 -0
- hpcmon/clusters/meluxina.py +235 -0
- hpcmon/clusters/slurm.py +89 -0
- hpcmon/config.py +347 -0
- hpcmon/models.py +319 -0
- hpcmon/render.py +420 -0
- hpcmon/runner.py +287 -0
- hpcmon/units.py +185 -0
- hpcmon-0.1.0.dist-info/METADATA +431 -0
- hpcmon-0.1.0.dist-info/RECORD +21 -0
- hpcmon-0.1.0.dist-info/WHEEL +5 -0
- hpcmon-0.1.0.dist-info/entry_points.txt +2 -0
- hpcmon-0.1.0.dist-info/licenses/LICENSE +21 -0
- hpcmon-0.1.0.dist-info/top_level.txt +1 -0
hpcmon/__init__.py
ADDED
hpcmon/__main__.py
ADDED
hpcmon/cli.py
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"""Command line interface of hpcmon."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import contextlib
|
|
7
|
+
import dataclasses
|
|
8
|
+
import datetime as dt
|
|
9
|
+
import logging
|
|
10
|
+
import math
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Iterator, List, Optional, Sequence
|
|
16
|
+
|
|
17
|
+
from rich.console import Console
|
|
18
|
+
from rich.markup import escape
|
|
19
|
+
from rich.text import Text
|
|
20
|
+
from rich.table import Table
|
|
21
|
+
|
|
22
|
+
from hpcmon import __version__
|
|
23
|
+
from hpcmon.clusters import CLUSTER_TYPES, Cluster, create_cluster
|
|
24
|
+
from hpcmon.config import (
|
|
25
|
+
CONFIG_ENV_VAR,
|
|
26
|
+
SECTIONS,
|
|
27
|
+
Config,
|
|
28
|
+
ConfigError,
|
|
29
|
+
default_config_path,
|
|
30
|
+
load_config,
|
|
31
|
+
write_default_config,
|
|
32
|
+
)
|
|
33
|
+
from hpcmon.models import ClusterReport
|
|
34
|
+
from hpcmon.render import render_reports, reports_to_json
|
|
35
|
+
from hpcmon.runner import terminate_all
|
|
36
|
+
|
|
37
|
+
#: Exit status when every cluster was queried successfully.
|
|
38
|
+
EXIT_OK = 0
|
|
39
|
+
#: Exit status when the data of at least one cluster is incomplete.
|
|
40
|
+
EXIT_CLUSTER_ERROR = 1
|
|
41
|
+
#: Exit status for invalid usage or configuration.
|
|
42
|
+
EXIT_USAGE = 2
|
|
43
|
+
#: Exit status when interrupted with Ctrl-C (128 + SIGINT).
|
|
44
|
+
EXIT_INTERRUPTED = 130
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
48
|
+
"""Create the parser of the command line arguments."""
|
|
49
|
+
parser = argparse.ArgumentParser(
|
|
50
|
+
prog="hpcmon",
|
|
51
|
+
description=(
|
|
52
|
+
"Display the remaining compute budgets, the storage quotas and the running jobs "
|
|
53
|
+
"of your projects on EuroHPC clusters (LUMI, JUPITER, Leonardo, MeluXina, ...)."
|
|
54
|
+
),
|
|
55
|
+
epilog=(
|
|
56
|
+
"By default all the enabled clusters of the configuration file are queried and all "
|
|
57
|
+
"sections are shown. Configuration file: $HPCMON_CONFIG or ~/.config/hpcmon/config.toml."
|
|
58
|
+
),
|
|
59
|
+
)
|
|
60
|
+
parser.add_argument("clusters", nargs="*", metavar="CLUSTER", help="names of the clusters to query (default: all enabled)")
|
|
61
|
+
parser.add_argument("-c", "--config", type=Path, metavar="PATH", help="configuration file to use")
|
|
62
|
+
sections = parser.add_argument_group("sections (default: all)")
|
|
63
|
+
sections.add_argument("--compute", action="store_true", help="show the compute allocations")
|
|
64
|
+
sections.add_argument("--storage", action="store_true", help="show the storage quotas")
|
|
65
|
+
sections.add_argument("--jobs", action="store_true", help="show the jobs in the queue")
|
|
66
|
+
parser.add_argument("--json", action="store_true", help="print the results as JSON")
|
|
67
|
+
parser.add_argument("-t", "--timeout", type=float, metavar="SECONDS", help="maximum time to wait for each cluster")
|
|
68
|
+
parser.add_argument("--sequential", action="store_true", help="query the clusters one after the other")
|
|
69
|
+
parser.add_argument("--no-color", action="store_true", help="disable colours")
|
|
70
|
+
parser.add_argument("--debug", action="store_true", help="log the raw output of the remote commands")
|
|
71
|
+
parser.add_argument("--init-config", action="store_true", help="write a default configuration file and exit")
|
|
72
|
+
parser.add_argument("--force", action="store_true", help="with --init-config, overwrite an existing file")
|
|
73
|
+
parser.add_argument("--show-config", action="store_true", help="show the configured clusters and exit")
|
|
74
|
+
parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}")
|
|
75
|
+
return parser
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def select_clusters(config: Config, names: Sequence[str]) -> List[str]:
|
|
79
|
+
"""Return the names of the clusters to query.
|
|
80
|
+
|
|
81
|
+
:param names: Names given on the command line; when empty, all the
|
|
82
|
+
enabled clusters are selected.
|
|
83
|
+
:raises ConfigError: for unknown names.
|
|
84
|
+
"""
|
|
85
|
+
if not names:
|
|
86
|
+
return [name for name, cluster in config.clusters.items() if cluster.enabled]
|
|
87
|
+
unknown = [name for name in names if name not in config.clusters]
|
|
88
|
+
if unknown:
|
|
89
|
+
available = ", ".join(config.clusters) or "none"
|
|
90
|
+
raise ConfigError(f"unknown cluster(s): {', '.join(unknown)} (configured: {available})")
|
|
91
|
+
return list(dict.fromkeys(names))
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def collect_reports(
|
|
95
|
+
clusters: Sequence[Cluster], sections: Sequence[str], parallel: bool, now: dt.datetime
|
|
96
|
+
) -> Iterator[ClusterReport]:
|
|
97
|
+
"""Query ``clusters`` and yield their reports in the same order.
|
|
98
|
+
|
|
99
|
+
With ``parallel``, all the clusters are queried at the same time and each
|
|
100
|
+
report is yielded as soon as it and all the preceding ones are complete.
|
|
101
|
+
"""
|
|
102
|
+
if not parallel or len(clusters) < 2:
|
|
103
|
+
for cluster in clusters:
|
|
104
|
+
yield _collect(cluster, sections, now)
|
|
105
|
+
return
|
|
106
|
+
pool = ThreadPoolExecutor(max_workers=len(clusters))
|
|
107
|
+
try:
|
|
108
|
+
futures = [pool.submit(_collect, cluster, sections, now) for cluster in clusters]
|
|
109
|
+
for future in futures:
|
|
110
|
+
yield future.result()
|
|
111
|
+
finally:
|
|
112
|
+
# Don't block (e.g. on Ctrl-C) until the remaining sessions end.
|
|
113
|
+
pool.shutdown(wait=False, cancel_futures=True)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _collect(cluster: Cluster, sections: Sequence[str], now: dt.datetime) -> ClusterReport:
|
|
117
|
+
try:
|
|
118
|
+
return cluster.collect(sections, now=now)
|
|
119
|
+
except Exception as exc: # noqa: BLE001 - report bugs per cluster rather than crash
|
|
120
|
+
logging.getLogger(__name__).debug("%s: collection failed", cluster.name, exc_info=True)
|
|
121
|
+
report = ClusterReport(
|
|
122
|
+
name=cluster.name, system=cluster.display_name, gpus=cluster.gpu_label, sections=list(sections),
|
|
123
|
+
collected_at=now,
|
|
124
|
+
)
|
|
125
|
+
report.errors.append(f"internal error ({type(exc).__name__}: {exc})")
|
|
126
|
+
report.failed_sections = list(sections)
|
|
127
|
+
return report
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@contextlib.contextmanager
|
|
131
|
+
def preserve_terminal() -> Iterator[None]:
|
|
132
|
+
"""Restore the terminal settings on exit.
|
|
133
|
+
|
|
134
|
+
An SSH command killed while prompting for a password (e.g. on timeout)
|
|
135
|
+
can leave the terminal with echo disabled; this undoes that.
|
|
136
|
+
"""
|
|
137
|
+
try:
|
|
138
|
+
import termios
|
|
139
|
+
except ImportError: # pragma: no cover - not available on Windows
|
|
140
|
+
yield
|
|
141
|
+
return
|
|
142
|
+
saved: Any = None
|
|
143
|
+
if sys.stdin is not None and sys.stdin.isatty():
|
|
144
|
+
try:
|
|
145
|
+
saved = termios.tcgetattr(sys.stdin.fileno())
|
|
146
|
+
except (termios.error, OSError, ValueError):
|
|
147
|
+
saved = None
|
|
148
|
+
try:
|
|
149
|
+
yield
|
|
150
|
+
finally:
|
|
151
|
+
if saved is not None:
|
|
152
|
+
try:
|
|
153
|
+
termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, saved)
|
|
154
|
+
except (termios.error, OSError, ValueError):
|
|
155
|
+
pass
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def show_config(config: Config, console: Console) -> None:
|
|
159
|
+
"""Print the configured clusters."""
|
|
160
|
+
source = str(config.path) if config.path else f"built-in defaults (no file at {default_config_path()})"
|
|
161
|
+
console.print(Text(f"Configuration: {source}"))
|
|
162
|
+
table = Table("Name", "Type", "SSH command", "Enabled", "Timeout", "Projects")
|
|
163
|
+
for cluster in config.clusters.values():
|
|
164
|
+
table.add_row(
|
|
165
|
+
Text(cluster.name),
|
|
166
|
+
Text(cluster.type),
|
|
167
|
+
Text(" ".join(cluster.ssh)),
|
|
168
|
+
Text("yes" if cluster.enabled else "no"),
|
|
169
|
+
Text(f"{cluster.timeout:g} s"),
|
|
170
|
+
Text(", ".join(cluster.projects) if cluster.projects is not None else "all"),
|
|
171
|
+
)
|
|
172
|
+
console.print(table)
|
|
173
|
+
console.print(Text(f"Supported types: {', '.join(CLUSTER_TYPES)}"))
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
177
|
+
"""Entry point of the ``hpcmon`` command."""
|
|
178
|
+
parser = build_parser()
|
|
179
|
+
# Intermixed parsing accepts cluster names after options (hpcmon --jobs lumi).
|
|
180
|
+
args = parser.parse_intermixed_args(argv)
|
|
181
|
+
sections = [section for section in SECTIONS if getattr(args, section)]
|
|
182
|
+
if args.force and not args.init_config:
|
|
183
|
+
parser.error("--force can only be used with --init-config")
|
|
184
|
+
if args.init_config and (args.clusters or sections or args.show_config or args.json):
|
|
185
|
+
parser.error("--init-config cannot be combined with clusters, sections, --show-config or --json")
|
|
186
|
+
console = Console(no_color=args.no_color, highlight=False)
|
|
187
|
+
err_console = Console(stderr=True, no_color=args.no_color, highlight=False)
|
|
188
|
+
if args.debug:
|
|
189
|
+
logging.basicConfig(level=logging.DEBUG, format="%(levelname)s %(name)s: %(message)s")
|
|
190
|
+
|
|
191
|
+
if args.init_config:
|
|
192
|
+
try:
|
|
193
|
+
path = write_default_config(args.config or default_config_path(), force=args.force)
|
|
194
|
+
except ConfigError as exc:
|
|
195
|
+
err_console.print(f"[bold red]error:[/] {escape(str(exc))}")
|
|
196
|
+
return EXIT_USAGE
|
|
197
|
+
console.print(Text(f"Configuration written to {path}"))
|
|
198
|
+
return EXIT_OK
|
|
199
|
+
|
|
200
|
+
try:
|
|
201
|
+
explicit = args.config is not None or bool(os.environ.get(CONFIG_ENV_VAR))
|
|
202
|
+
config = load_config(args.config, required=explicit)
|
|
203
|
+
names = select_clusters(config, args.clusters)
|
|
204
|
+
except ConfigError as exc:
|
|
205
|
+
err_console.print(f"[bold red]error:[/] {escape(str(exc))}")
|
|
206
|
+
return EXIT_USAGE
|
|
207
|
+
|
|
208
|
+
if args.show_config:
|
|
209
|
+
show_config(config, console)
|
|
210
|
+
return EXIT_OK
|
|
211
|
+
if not names:
|
|
212
|
+
err_console.print("[bold red]error:[/] no cluster is enabled in the configuration")
|
|
213
|
+
return EXIT_USAGE
|
|
214
|
+
if args.timeout is not None and not (math.isfinite(args.timeout) and args.timeout > 0):
|
|
215
|
+
err_console.print("[bold red]error:[/] the timeout must be positive")
|
|
216
|
+
return EXIT_USAGE
|
|
217
|
+
|
|
218
|
+
sections = sections or list(SECTIONS)
|
|
219
|
+
clusters = []
|
|
220
|
+
for name in names:
|
|
221
|
+
cluster_config = config.clusters[name]
|
|
222
|
+
if args.timeout is not None:
|
|
223
|
+
cluster_config = dataclasses.replace(cluster_config, timeout=args.timeout)
|
|
224
|
+
clusters.append(create_cluster(cluster_config))
|
|
225
|
+
|
|
226
|
+
now = dt.datetime.now().astimezone()
|
|
227
|
+
parallel = config.parallel and not args.sequential and len(clusters) > 1
|
|
228
|
+
reports = collect_reports(clusters, sections, parallel=parallel, now=now)
|
|
229
|
+
status_console = err_console if args.json else console
|
|
230
|
+
# No spinner when querying sequentially (the mode meant for clusters that
|
|
231
|
+
# prompt for a password or OTP): its redraws would hide the SSH prompts.
|
|
232
|
+
spinner = (
|
|
233
|
+
status_console.status(Text(f"Querying {', '.join(names)}..."))
|
|
234
|
+
if parallel
|
|
235
|
+
else contextlib.nullcontext()
|
|
236
|
+
)
|
|
237
|
+
try:
|
|
238
|
+
with preserve_terminal(), spinner:
|
|
239
|
+
if args.json:
|
|
240
|
+
collected = list(reports)
|
|
241
|
+
else:
|
|
242
|
+
collected = render_reports(reports, console, today=now.date())
|
|
243
|
+
except KeyboardInterrupt:
|
|
244
|
+
terminate_all()
|
|
245
|
+
err_console.print("[bold red]interrupted[/]")
|
|
246
|
+
return EXIT_INTERRUPTED
|
|
247
|
+
if args.json:
|
|
248
|
+
sys.stdout.write(reports_to_json(collected, generated_at=now) + "\n")
|
|
249
|
+
return EXIT_OK if all(report.ok for report in collected) else EXIT_CLUSTER_ERROR
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
if __name__ == "__main__": # pragma: no cover
|
|
253
|
+
sys.exit(main())
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Cluster-specific interfaces and the registry of supported cluster types."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Dict, Optional, Type
|
|
6
|
+
|
|
7
|
+
from hpcmon.clusters.base import Cluster, CollectionError
|
|
8
|
+
from hpcmon.clusters.jupiter import JupiterCluster
|
|
9
|
+
from hpcmon.clusters.leonardo import LeonardoCluster
|
|
10
|
+
from hpcmon.clusters.lumi import LumiCluster
|
|
11
|
+
from hpcmon.clusters.meluxina import MeluxinaCluster
|
|
12
|
+
from hpcmon.clusters.slurm import GenericSlurmCluster, SlurmCluster
|
|
13
|
+
from hpcmon.config import ClusterConfig
|
|
14
|
+
from hpcmon.runner import SSHRunner
|
|
15
|
+
|
|
16
|
+
#: All supported cluster types, by the ``type`` used in the configuration.
|
|
17
|
+
CLUSTER_TYPES: Dict[str, Type[Cluster]] = {
|
|
18
|
+
cls.type_name: cls
|
|
19
|
+
for cls in (LumiCluster, JupiterCluster, LeonardoCluster, MeluxinaCluster, GenericSlurmCluster)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def create_cluster(config: ClusterConfig, runner: Optional[SSHRunner] = None) -> Cluster:
|
|
24
|
+
"""Instantiate the interface for the cluster described by ``config``.
|
|
25
|
+
|
|
26
|
+
:raises KeyError: if the type of the cluster is not supported.
|
|
27
|
+
"""
|
|
28
|
+
try:
|
|
29
|
+
cls = CLUSTER_TYPES[config.type]
|
|
30
|
+
except KeyError:
|
|
31
|
+
raise KeyError(f"unsupported cluster type: {config.type!r}") from None
|
|
32
|
+
return cls(config, runner=runner)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"CLUSTER_TYPES",
|
|
37
|
+
"Cluster",
|
|
38
|
+
"CollectionError",
|
|
39
|
+
"GenericSlurmCluster",
|
|
40
|
+
"JupiterCluster",
|
|
41
|
+
"LeonardoCluster",
|
|
42
|
+
"LumiCluster",
|
|
43
|
+
"MeluxinaCluster",
|
|
44
|
+
"SlurmCluster",
|
|
45
|
+
"create_cluster",
|
|
46
|
+
]
|
hpcmon/clusters/base.py
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"""Abstract interface implemented by every supported cluster."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime as dt
|
|
6
|
+
import logging
|
|
7
|
+
from abc import ABC, abstractmethod
|
|
8
|
+
from typing import Callable, ClassVar, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, TypeVar
|
|
9
|
+
|
|
10
|
+
from hpcmon.config import SECTIONS, ClusterConfig
|
|
11
|
+
from hpcmon.models import ClusterReport, ComputeBudget, Job, StorageQuota
|
|
12
|
+
from hpcmon.runner import TIMEOUT_EXIT_CODE, CommandResult, RemoteError, SSHRunner, last_line
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
#: The results of the remote commands, by command key.
|
|
17
|
+
Results = Mapping[str, CommandResult]
|
|
18
|
+
|
|
19
|
+
#: Remote command printing the home directory of the user.
|
|
20
|
+
HOME_COMMAND = 'printf "%s\\n" "$HOME"'
|
|
21
|
+
|
|
22
|
+
#: Order in which the commands of the sections run: the fast ones first, so
|
|
23
|
+
#: that a slow storage tool cannot use up the time limit of the others.
|
|
24
|
+
EXECUTION_ORDER = ("jobs", "compute", "storage")
|
|
25
|
+
|
|
26
|
+
T = TypeVar("T")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class CollectionError(Exception):
|
|
30
|
+
"""Raised when the output of a remote command is missing or unusable."""
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class Cluster(ABC):
|
|
34
|
+
"""Base class of the cluster-specific interfaces.
|
|
35
|
+
|
|
36
|
+
A cluster declares the remote commands needed for every section of
|
|
37
|
+
information (compute budgets, storage quotas and jobs) and parses their
|
|
38
|
+
output into the cluster-independent model of :mod:`hpcmon.models`. All the
|
|
39
|
+
commands of a query are executed through a single SSH session.
|
|
40
|
+
|
|
41
|
+
Parsers raise :class:`CollectionError` (or any other exception) when the
|
|
42
|
+
data of their section cannot be obtained; this is reported for that
|
|
43
|
+
section only, without affecting the others. Non-fatal problems can be
|
|
44
|
+
reported with :meth:`warn`.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
#: Identifier of the cluster type, used for ``type`` in the configuration.
|
|
48
|
+
type_name: ClassVar[str] = ""
|
|
49
|
+
#: Human-readable name of the system.
|
|
50
|
+
display_name: ClassVar[str] = ""
|
|
51
|
+
#: Sections of information that this kind of cluster can provide.
|
|
52
|
+
supported_sections: ClassVar[Tuple[str, ...]] = SECTIONS
|
|
53
|
+
#: The GPUs of a GPU node, e.g. ``"4x NVIDIA A100 64 GB"`` (``None`` if
|
|
54
|
+
#: unknown); the ``gpus`` setting replaces it.
|
|
55
|
+
gpus: ClassVar[Optional[str]] = None
|
|
56
|
+
#: Estimated GPU-hours bought by one unit of a budget, by the
|
|
57
|
+
#: ``(resource, unit)`` of the budget (the resource is compared without
|
|
58
|
+
#: case, the unit exactly). Budgets missing here (CPU-only partitions,
|
|
59
|
+
#: storage billing, unknown units) get no estimate. The rates assume that
|
|
60
|
+
#: jobs use all the GPUs of the nodes they are charged for: on shared
|
|
61
|
+
#: nodes, no more than the share of cores and memory of their GPUs.
|
|
62
|
+
gpu_rates: ClassVar[Mapping[Tuple[str, str], float]] = {}
|
|
63
|
+
|
|
64
|
+
def __init__(self, config: ClusterConfig, runner: Optional[SSHRunner] = None) -> None:
|
|
65
|
+
self.config = config
|
|
66
|
+
self.runner = runner or SSHRunner(config.ssh, timeout=config.timeout, login_shell=config.login_shell)
|
|
67
|
+
self._commands: Dict[str, str] = {}
|
|
68
|
+
self._warnings: List[str] = []
|
|
69
|
+
|
|
70
|
+
@property
|
|
71
|
+
def name(self) -> str:
|
|
72
|
+
"""Name of the cluster in the configuration."""
|
|
73
|
+
return self.config.name
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def gpu_label(self) -> Optional[str]:
|
|
77
|
+
"""Description of the GPUs of a node: the setting, else the built-in one."""
|
|
78
|
+
label = self.config.gpus if self.config.gpus is not None else self.gpus
|
|
79
|
+
return (label or "").strip() or None
|
|
80
|
+
|
|
81
|
+
def gpu_rate(self, budget: ComputeBudget) -> Optional[float]:
|
|
82
|
+
"""GPU-hours bought by one unit of ``budget`` (``None``: not convertible).
|
|
83
|
+
|
|
84
|
+
A rate in the ``gpu_hours_per_unit`` setting takes precedence (0
|
|
85
|
+
disables the estimate); otherwise :attr:`gpu_rates` is used.
|
|
86
|
+
"""
|
|
87
|
+
configured = self.config.gpu_rate_for(budget.resource)
|
|
88
|
+
if configured is not None:
|
|
89
|
+
return configured or None
|
|
90
|
+
for (resource, unit), rate in self.gpu_rates.items():
|
|
91
|
+
if resource.lower() == budget.resource.lower() and unit == budget.unit:
|
|
92
|
+
return rate
|
|
93
|
+
return None
|
|
94
|
+
|
|
95
|
+
# -- interface to implement -------------------------------------------
|
|
96
|
+
|
|
97
|
+
@abstractmethod
|
|
98
|
+
def compute_commands(self) -> Dict[str, str]:
|
|
99
|
+
"""Remote commands (by unique key) needed by :meth:`parse_compute`."""
|
|
100
|
+
|
|
101
|
+
@abstractmethod
|
|
102
|
+
def storage_commands(self) -> Dict[str, str]:
|
|
103
|
+
"""Remote commands (by unique key) needed by :meth:`parse_storage`."""
|
|
104
|
+
|
|
105
|
+
@abstractmethod
|
|
106
|
+
def jobs_commands(self) -> Dict[str, str]:
|
|
107
|
+
"""Remote commands (by unique key) needed by :meth:`parse_jobs`."""
|
|
108
|
+
|
|
109
|
+
@abstractmethod
|
|
110
|
+
def parse_compute(self, results: Results, today: dt.date) -> List[ComputeBudget]:
|
|
111
|
+
"""Extract the compute budgets of the user's projects."""
|
|
112
|
+
|
|
113
|
+
@abstractmethod
|
|
114
|
+
def parse_storage(self, results: Results, today: dt.date) -> List[StorageQuota]:
|
|
115
|
+
"""Extract the storage quotas of the user and their projects."""
|
|
116
|
+
|
|
117
|
+
@abstractmethod
|
|
118
|
+
def parse_jobs(self, results: Results, today: dt.date) -> List[Job]:
|
|
119
|
+
"""Extract the jobs of the user."""
|
|
120
|
+
|
|
121
|
+
# -- helpers for implementations --------------------------------------
|
|
122
|
+
|
|
123
|
+
def output(self, results: Results, key: str, *, check: bool = True) -> str:
|
|
124
|
+
"""Return the standard output of the command ``key``.
|
|
125
|
+
|
|
126
|
+
:param check: If true, a non-zero exit status is an error.
|
|
127
|
+
:raises CollectionError: if the command did not run, timed out or
|
|
128
|
+
(with ``check``) failed.
|
|
129
|
+
"""
|
|
130
|
+
command = self._commands.get(key, key)
|
|
131
|
+
result = results.get(key)
|
|
132
|
+
if result is None:
|
|
133
|
+
raise CollectionError(f"no output from `{command}`")
|
|
134
|
+
if result.returncode is None:
|
|
135
|
+
raise CollectionError(f"`{command}` did not complete: {result.reason or 'no exit status'}")
|
|
136
|
+
if result.returncode == TIMEOUT_EXIT_CODE:
|
|
137
|
+
raise CollectionError(f"`{command}` timed out (time limit of {self.config.timeout:g} s reached)")
|
|
138
|
+
if check and result.returncode != 0:
|
|
139
|
+
raise CollectionError(f"`{command}` failed (exit status {result.returncode}){_detail(result)}")
|
|
140
|
+
return result.stdout
|
|
141
|
+
|
|
142
|
+
def warn(self, message: str) -> None:
|
|
143
|
+
"""Report a non-fatal problem in the current collection."""
|
|
144
|
+
self._warnings.append(message)
|
|
145
|
+
|
|
146
|
+
# -- collection -------------------------------------------------------
|
|
147
|
+
|
|
148
|
+
def commands(self, sections: Iterable[str] = SECTIONS) -> Dict[str, str]:
|
|
149
|
+
"""All the remote commands needed for ``sections``, in execution order."""
|
|
150
|
+
providers = {
|
|
151
|
+
"compute": self.compute_commands,
|
|
152
|
+
"storage": self.storage_commands,
|
|
153
|
+
"jobs": self.jobs_commands,
|
|
154
|
+
}
|
|
155
|
+
wanted = set(sections)
|
|
156
|
+
commands: Dict[str, str] = {}
|
|
157
|
+
for section in EXECUTION_ORDER:
|
|
158
|
+
if section in wanted:
|
|
159
|
+
for key, command in providers[section]().items():
|
|
160
|
+
commands.setdefault(key, command)
|
|
161
|
+
return commands
|
|
162
|
+
|
|
163
|
+
def collect(
|
|
164
|
+
self,
|
|
165
|
+
sections: Sequence[str] = SECTIONS,
|
|
166
|
+
today: Optional[dt.date] = None,
|
|
167
|
+
now: Optional[dt.datetime] = None,
|
|
168
|
+
) -> ClusterReport:
|
|
169
|
+
"""Query the cluster and return everything found for ``sections``.
|
|
170
|
+
|
|
171
|
+
This never raises for problems with the cluster: they are recorded in
|
|
172
|
+
the ``errors`` of the returned report instead.
|
|
173
|
+
"""
|
|
174
|
+
unknown = [section for section in sections if section not in SECTIONS]
|
|
175
|
+
if unknown:
|
|
176
|
+
raise ValueError(f"unknown section(s): {', '.join(unknown)}")
|
|
177
|
+
sections = [section for section in SECTIONS if section in sections and section in self.supported_sections]
|
|
178
|
+
now = now or dt.datetime.now().astimezone()
|
|
179
|
+
today = today or now.date()
|
|
180
|
+
report = ClusterReport(
|
|
181
|
+
name=self.name, system=self.display_name, gpus=self.gpu_label, sections=sections, collected_at=now
|
|
182
|
+
)
|
|
183
|
+
self._commands = self.commands(sections)
|
|
184
|
+
self._warnings = []
|
|
185
|
+
if self.config.gpu_hours_per_unit and "compute" not in self.supported_sections:
|
|
186
|
+
self.warn("the gpu_hours_per_unit setting is ignored: this type of cluster reports no compute budgets")
|
|
187
|
+
try:
|
|
188
|
+
results = self.runner.run(self._commands)
|
|
189
|
+
except RemoteError as exc:
|
|
190
|
+
report.errors.append(str(exc))
|
|
191
|
+
report.failed_sections = list(sections)
|
|
192
|
+
return report
|
|
193
|
+
for key, result in results.items():
|
|
194
|
+
logger.debug(
|
|
195
|
+
"%s: `%s` exited with %s\n--- stdout ---\n%s\n--- stderr ---\n%s",
|
|
196
|
+
self.name, self._commands[key], result.returncode, result.stdout, result.stderr,
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
# Names of the projects known to the cluster, to detect typos in the
|
|
200
|
+
# "projects" setting (jobs are not used: the queue may be empty).
|
|
201
|
+
seen: Set[str] = set()
|
|
202
|
+
if "compute" in sections:
|
|
203
|
+
budgets = self._guard(report, "compute", self.parse_compute, results, today) or []
|
|
204
|
+
seen.update(name for budget in budgets for name in (budget.project, budget.account) if name)
|
|
205
|
+
report.budgets = [self._complete(budget) for budget in budgets if self._wants_budget(budget)]
|
|
206
|
+
if "compute" not in report.failed_sections:
|
|
207
|
+
self._check_gpu_rates(budgets)
|
|
208
|
+
if "storage" in sections:
|
|
209
|
+
quotas = self._guard(report, "storage", self.parse_storage, results, today) or []
|
|
210
|
+
seen.update(quota.project for quota in quotas if quota.project)
|
|
211
|
+
report.storage = [quota for quota in quotas if self.config.wants_project(quota.project)]
|
|
212
|
+
if "jobs" in sections:
|
|
213
|
+
jobs = self._guard(report, "jobs", self.parse_jobs, results, today)
|
|
214
|
+
if jobs is not None:
|
|
215
|
+
report.jobs = [job for job in jobs if self.config.wants_project(job.account or None)]
|
|
216
|
+
hidden = len(jobs) - len(report.jobs)
|
|
217
|
+
if hidden:
|
|
218
|
+
self.warn(f"{hidden} job(s) of other accounts hidden by the 'projects' setting")
|
|
219
|
+
checked = {"compute", "storage"} & set(sections)
|
|
220
|
+
if self.config.projects is not None and checked and not report.failed_sections:
|
|
221
|
+
known = {name.lower() for name in seen}
|
|
222
|
+
missing = [name for name in self.config.projects if name.lower() not in known]
|
|
223
|
+
if missing:
|
|
224
|
+
self.warn(f"project(s) not found on this cluster: {', '.join(missing)}")
|
|
225
|
+
report.warnings.extend(self._warnings)
|
|
226
|
+
return report
|
|
227
|
+
|
|
228
|
+
def _check_gpu_rates(self, budgets: List[ComputeBudget]) -> None:
|
|
229
|
+
"""Warn about rates of the ``gpu_hours_per_unit`` setting that apply to
|
|
230
|
+
no budget of the cluster (e.g. typos in the resource names)."""
|
|
231
|
+
resources = sorted({budget.resource for budget in budgets})
|
|
232
|
+
unused = [name for name in self.config.gpu_hours_per_unit if name.lower() not in {r.lower() for r in resources}]
|
|
233
|
+
if unused:
|
|
234
|
+
known = f" (budget resources: {', '.join(resources)})" if resources else ""
|
|
235
|
+
self.warn(f"gpu_hours_per_unit: no budget of resource {', '.join(unused)} on this cluster{known}")
|
|
236
|
+
|
|
237
|
+
def _wants_budget(self, budget: ComputeBudget) -> bool:
|
|
238
|
+
if self.config.wants_project(budget.project):
|
|
239
|
+
return True
|
|
240
|
+
return budget.account is not None and self.config.wants_project(budget.account)
|
|
241
|
+
|
|
242
|
+
def _guard(
|
|
243
|
+
self,
|
|
244
|
+
report: ClusterReport,
|
|
245
|
+
section: str,
|
|
246
|
+
parser: Callable[[Results, dt.date], List[T]],
|
|
247
|
+
results: Results,
|
|
248
|
+
today: dt.date,
|
|
249
|
+
) -> Optional[List[T]]:
|
|
250
|
+
try:
|
|
251
|
+
return parser(results, today)
|
|
252
|
+
except CollectionError as exc:
|
|
253
|
+
report.errors.append(f"{section}: {exc}")
|
|
254
|
+
except Exception as exc: # noqa: BLE001 - one broken parser must not hide the rest
|
|
255
|
+
logger.debug("%s: cannot parse %s", self.name, section, exc_info=True)
|
|
256
|
+
report.errors.append(f"{section}: unexpected output ({type(exc).__name__}: {exc})")
|
|
257
|
+
report.failed_sections.append(section)
|
|
258
|
+
return None
|
|
259
|
+
|
|
260
|
+
def _complete(self, budget: ComputeBudget) -> ComputeBudget:
|
|
261
|
+
if budget.end is None:
|
|
262
|
+
budget.end = self.config.end_date_for(budget.project)
|
|
263
|
+
# A rate set by the parser is kept, unless the configuration has one.
|
|
264
|
+
if budget.gpu_hours_per_unit is None or self.config.gpu_rate_for(budget.resource) is not None:
|
|
265
|
+
budget.gpu_hours_per_unit = self.gpu_rate(budget)
|
|
266
|
+
return budget
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _detail(result: CommandResult) -> str:
|
|
270
|
+
line = last_line(result.stderr, result.stdout)
|
|
271
|
+
return f": {line}" if line else ""
|