hpcmon 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hpcmon/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """hpcmon: monitor your compute budgets, storage quotas and jobs on EuroHPC clusters."""
2
+
3
+ __version__ = "0.1.0"
hpcmon/__main__.py ADDED
@@ -0,0 +1,8 @@
1
+ """Allow running hpcmon as ``python -m hpcmon``."""
2
+
3
+ import sys
4
+
5
+ from hpcmon.cli import main
6
+
7
+ if __name__ == "__main__":
8
+ sys.exit(main())
hpcmon/cli.py ADDED
@@ -0,0 +1,253 @@
1
+ """Command line interface of hpcmon."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import contextlib
7
+ import dataclasses
8
+ import datetime as dt
9
+ import logging
10
+ import math
11
+ import os
12
+ import sys
13
+ from concurrent.futures import ThreadPoolExecutor
14
+ from pathlib import Path
15
+ from typing import Any, Iterator, List, Optional, Sequence
16
+
17
+ from rich.console import Console
18
+ from rich.markup import escape
19
+ from rich.text import Text
20
+ from rich.table import Table
21
+
22
+ from hpcmon import __version__
23
+ from hpcmon.clusters import CLUSTER_TYPES, Cluster, create_cluster
24
+ from hpcmon.config import (
25
+ CONFIG_ENV_VAR,
26
+ SECTIONS,
27
+ Config,
28
+ ConfigError,
29
+ default_config_path,
30
+ load_config,
31
+ write_default_config,
32
+ )
33
+ from hpcmon.models import ClusterReport
34
+ from hpcmon.render import render_reports, reports_to_json
35
+ from hpcmon.runner import terminate_all
36
+
37
+ #: Exit status when every cluster was queried successfully.
38
+ EXIT_OK = 0
39
+ #: Exit status when the data of at least one cluster is incomplete.
40
+ EXIT_CLUSTER_ERROR = 1
41
+ #: Exit status for invalid usage or configuration.
42
+ EXIT_USAGE = 2
43
+ #: Exit status when interrupted with Ctrl-C (128 + SIGINT).
44
+ EXIT_INTERRUPTED = 130
45
+
46
+
47
+ def build_parser() -> argparse.ArgumentParser:
48
+ """Create the parser of the command line arguments."""
49
+ parser = argparse.ArgumentParser(
50
+ prog="hpcmon",
51
+ description=(
52
+ "Display the remaining compute budgets, the storage quotas and the running jobs "
53
+ "of your projects on EuroHPC clusters (LUMI, JUPITER, Leonardo, MeluXina, ...)."
54
+ ),
55
+ epilog=(
56
+ "By default all the enabled clusters of the configuration file are queried and all "
57
+ "sections are shown. Configuration file: $HPCMON_CONFIG or ~/.config/hpcmon/config.toml."
58
+ ),
59
+ )
60
+ parser.add_argument("clusters", nargs="*", metavar="CLUSTER", help="names of the clusters to query (default: all enabled)")
61
+ parser.add_argument("-c", "--config", type=Path, metavar="PATH", help="configuration file to use")
62
+ sections = parser.add_argument_group("sections (default: all)")
63
+ sections.add_argument("--compute", action="store_true", help="show the compute allocations")
64
+ sections.add_argument("--storage", action="store_true", help="show the storage quotas")
65
+ sections.add_argument("--jobs", action="store_true", help="show the jobs in the queue")
66
+ parser.add_argument("--json", action="store_true", help="print the results as JSON")
67
+ parser.add_argument("-t", "--timeout", type=float, metavar="SECONDS", help="maximum time to wait for each cluster")
68
+ parser.add_argument("--sequential", action="store_true", help="query the clusters one after the other")
69
+ parser.add_argument("--no-color", action="store_true", help="disable colours")
70
+ parser.add_argument("--debug", action="store_true", help="log the raw output of the remote commands")
71
+ parser.add_argument("--init-config", action="store_true", help="write a default configuration file and exit")
72
+ parser.add_argument("--force", action="store_true", help="with --init-config, overwrite an existing file")
73
+ parser.add_argument("--show-config", action="store_true", help="show the configured clusters and exit")
74
+ parser.add_argument("-V", "--version", action="version", version=f"%(prog)s {__version__}")
75
+ return parser
76
+
77
+
78
+ def select_clusters(config: Config, names: Sequence[str]) -> List[str]:
79
+ """Return the names of the clusters to query.
80
+
81
+ :param names: Names given on the command line; when empty, all the
82
+ enabled clusters are selected.
83
+ :raises ConfigError: for unknown names.
84
+ """
85
+ if not names:
86
+ return [name for name, cluster in config.clusters.items() if cluster.enabled]
87
+ unknown = [name for name in names if name not in config.clusters]
88
+ if unknown:
89
+ available = ", ".join(config.clusters) or "none"
90
+ raise ConfigError(f"unknown cluster(s): {', '.join(unknown)} (configured: {available})")
91
+ return list(dict.fromkeys(names))
92
+
93
+
94
+ def collect_reports(
95
+ clusters: Sequence[Cluster], sections: Sequence[str], parallel: bool, now: dt.datetime
96
+ ) -> Iterator[ClusterReport]:
97
+ """Query ``clusters`` and yield their reports in the same order.
98
+
99
+ With ``parallel``, all the clusters are queried at the same time and each
100
+ report is yielded as soon as it and all the preceding ones are complete.
101
+ """
102
+ if not parallel or len(clusters) < 2:
103
+ for cluster in clusters:
104
+ yield _collect(cluster, sections, now)
105
+ return
106
+ pool = ThreadPoolExecutor(max_workers=len(clusters))
107
+ try:
108
+ futures = [pool.submit(_collect, cluster, sections, now) for cluster in clusters]
109
+ for future in futures:
110
+ yield future.result()
111
+ finally:
112
+ # Don't block (e.g. on Ctrl-C) until the remaining sessions end.
113
+ pool.shutdown(wait=False, cancel_futures=True)
114
+
115
+
116
+ def _collect(cluster: Cluster, sections: Sequence[str], now: dt.datetime) -> ClusterReport:
117
+ try:
118
+ return cluster.collect(sections, now=now)
119
+ except Exception as exc: # noqa: BLE001 - report bugs per cluster rather than crash
120
+ logging.getLogger(__name__).debug("%s: collection failed", cluster.name, exc_info=True)
121
+ report = ClusterReport(
122
+ name=cluster.name, system=cluster.display_name, gpus=cluster.gpu_label, sections=list(sections),
123
+ collected_at=now,
124
+ )
125
+ report.errors.append(f"internal error ({type(exc).__name__}: {exc})")
126
+ report.failed_sections = list(sections)
127
+ return report
128
+
129
+
130
+ @contextlib.contextmanager
131
+ def preserve_terminal() -> Iterator[None]:
132
+ """Restore the terminal settings on exit.
133
+
134
+ An SSH command killed while prompting for a password (e.g. on timeout)
135
+ can leave the terminal with echo disabled; this undoes that.
136
+ """
137
+ try:
138
+ import termios
139
+ except ImportError: # pragma: no cover - not available on Windows
140
+ yield
141
+ return
142
+ saved: Any = None
143
+ if sys.stdin is not None and sys.stdin.isatty():
144
+ try:
145
+ saved = termios.tcgetattr(sys.stdin.fileno())
146
+ except (termios.error, OSError, ValueError):
147
+ saved = None
148
+ try:
149
+ yield
150
+ finally:
151
+ if saved is not None:
152
+ try:
153
+ termios.tcsetattr(sys.stdin.fileno(), termios.TCSADRAIN, saved)
154
+ except (termios.error, OSError, ValueError):
155
+ pass
156
+
157
+
158
+ def show_config(config: Config, console: Console) -> None:
159
+ """Print the configured clusters."""
160
+ source = str(config.path) if config.path else f"built-in defaults (no file at {default_config_path()})"
161
+ console.print(Text(f"Configuration: {source}"))
162
+ table = Table("Name", "Type", "SSH command", "Enabled", "Timeout", "Projects")
163
+ for cluster in config.clusters.values():
164
+ table.add_row(
165
+ Text(cluster.name),
166
+ Text(cluster.type),
167
+ Text(" ".join(cluster.ssh)),
168
+ Text("yes" if cluster.enabled else "no"),
169
+ Text(f"{cluster.timeout:g} s"),
170
+ Text(", ".join(cluster.projects) if cluster.projects is not None else "all"),
171
+ )
172
+ console.print(table)
173
+ console.print(Text(f"Supported types: {', '.join(CLUSTER_TYPES)}"))
174
+
175
+
176
+ def main(argv: Optional[Sequence[str]] = None) -> int:
177
+ """Entry point of the ``hpcmon`` command."""
178
+ parser = build_parser()
179
+ # Intermixed parsing accepts cluster names after options (hpcmon --jobs lumi).
180
+ args = parser.parse_intermixed_args(argv)
181
+ sections = [section for section in SECTIONS if getattr(args, section)]
182
+ if args.force and not args.init_config:
183
+ parser.error("--force can only be used with --init-config")
184
+ if args.init_config and (args.clusters or sections or args.show_config or args.json):
185
+ parser.error("--init-config cannot be combined with clusters, sections, --show-config or --json")
186
+ console = Console(no_color=args.no_color, highlight=False)
187
+ err_console = Console(stderr=True, no_color=args.no_color, highlight=False)
188
+ if args.debug:
189
+ logging.basicConfig(level=logging.DEBUG, format="%(levelname)s %(name)s: %(message)s")
190
+
191
+ if args.init_config:
192
+ try:
193
+ path = write_default_config(args.config or default_config_path(), force=args.force)
194
+ except ConfigError as exc:
195
+ err_console.print(f"[bold red]error:[/] {escape(str(exc))}")
196
+ return EXIT_USAGE
197
+ console.print(Text(f"Configuration written to {path}"))
198
+ return EXIT_OK
199
+
200
+ try:
201
+ explicit = args.config is not None or bool(os.environ.get(CONFIG_ENV_VAR))
202
+ config = load_config(args.config, required=explicit)
203
+ names = select_clusters(config, args.clusters)
204
+ except ConfigError as exc:
205
+ err_console.print(f"[bold red]error:[/] {escape(str(exc))}")
206
+ return EXIT_USAGE
207
+
208
+ if args.show_config:
209
+ show_config(config, console)
210
+ return EXIT_OK
211
+ if not names:
212
+ err_console.print("[bold red]error:[/] no cluster is enabled in the configuration")
213
+ return EXIT_USAGE
214
+ if args.timeout is not None and not (math.isfinite(args.timeout) and args.timeout > 0):
215
+ err_console.print("[bold red]error:[/] the timeout must be positive")
216
+ return EXIT_USAGE
217
+
218
+ sections = sections or list(SECTIONS)
219
+ clusters = []
220
+ for name in names:
221
+ cluster_config = config.clusters[name]
222
+ if args.timeout is not None:
223
+ cluster_config = dataclasses.replace(cluster_config, timeout=args.timeout)
224
+ clusters.append(create_cluster(cluster_config))
225
+
226
+ now = dt.datetime.now().astimezone()
227
+ parallel = config.parallel and not args.sequential and len(clusters) > 1
228
+ reports = collect_reports(clusters, sections, parallel=parallel, now=now)
229
+ status_console = err_console if args.json else console
230
+ # No spinner when querying sequentially (the mode meant for clusters that
231
+ # prompt for a password or OTP): its redraws would hide the SSH prompts.
232
+ spinner = (
233
+ status_console.status(Text(f"Querying {', '.join(names)}..."))
234
+ if parallel
235
+ else contextlib.nullcontext()
236
+ )
237
+ try:
238
+ with preserve_terminal(), spinner:
239
+ if args.json:
240
+ collected = list(reports)
241
+ else:
242
+ collected = render_reports(reports, console, today=now.date())
243
+ except KeyboardInterrupt:
244
+ terminate_all()
245
+ err_console.print("[bold red]interrupted[/]")
246
+ return EXIT_INTERRUPTED
247
+ if args.json:
248
+ sys.stdout.write(reports_to_json(collected, generated_at=now) + "\n")
249
+ return EXIT_OK if all(report.ok for report in collected) else EXIT_CLUSTER_ERROR
250
+
251
+
252
+ if __name__ == "__main__": # pragma: no cover
253
+ sys.exit(main())
@@ -0,0 +1,46 @@
1
+ """Cluster-specific interfaces and the registry of supported cluster types."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Dict, Optional, Type
6
+
7
+ from hpcmon.clusters.base import Cluster, CollectionError
8
+ from hpcmon.clusters.jupiter import JupiterCluster
9
+ from hpcmon.clusters.leonardo import LeonardoCluster
10
+ from hpcmon.clusters.lumi import LumiCluster
11
+ from hpcmon.clusters.meluxina import MeluxinaCluster
12
+ from hpcmon.clusters.slurm import GenericSlurmCluster, SlurmCluster
13
+ from hpcmon.config import ClusterConfig
14
+ from hpcmon.runner import SSHRunner
15
+
16
+ #: All supported cluster types, by the ``type`` used in the configuration.
17
+ CLUSTER_TYPES: Dict[str, Type[Cluster]] = {
18
+ cls.type_name: cls
19
+ for cls in (LumiCluster, JupiterCluster, LeonardoCluster, MeluxinaCluster, GenericSlurmCluster)
20
+ }
21
+
22
+
23
+ def create_cluster(config: ClusterConfig, runner: Optional[SSHRunner] = None) -> Cluster:
24
+ """Instantiate the interface for the cluster described by ``config``.
25
+
26
+ :raises KeyError: if the type of the cluster is not supported.
27
+ """
28
+ try:
29
+ cls = CLUSTER_TYPES[config.type]
30
+ except KeyError:
31
+ raise KeyError(f"unsupported cluster type: {config.type!r}") from None
32
+ return cls(config, runner=runner)
33
+
34
+
35
+ __all__ = [
36
+ "CLUSTER_TYPES",
37
+ "Cluster",
38
+ "CollectionError",
39
+ "GenericSlurmCluster",
40
+ "JupiterCluster",
41
+ "LeonardoCluster",
42
+ "LumiCluster",
43
+ "MeluxinaCluster",
44
+ "SlurmCluster",
45
+ "create_cluster",
46
+ ]
@@ -0,0 +1,271 @@
1
+ """Abstract interface implemented by every supported cluster."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import datetime as dt
6
+ import logging
7
+ from abc import ABC, abstractmethod
8
+ from typing import Callable, ClassVar, Dict, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, TypeVar
9
+
10
+ from hpcmon.config import SECTIONS, ClusterConfig
11
+ from hpcmon.models import ClusterReport, ComputeBudget, Job, StorageQuota
12
+ from hpcmon.runner import TIMEOUT_EXIT_CODE, CommandResult, RemoteError, SSHRunner, last_line
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ #: The results of the remote commands, by command key.
17
+ Results = Mapping[str, CommandResult]
18
+
19
+ #: Remote command printing the home directory of the user.
20
+ HOME_COMMAND = 'printf "%s\\n" "$HOME"'
21
+
22
+ #: Order in which the commands of the sections run: the fast ones first, so
23
+ #: that a slow storage tool cannot use up the time limit of the others.
24
+ EXECUTION_ORDER = ("jobs", "compute", "storage")
25
+
26
+ T = TypeVar("T")
27
+
28
+
29
+ class CollectionError(Exception):
30
+ """Raised when the output of a remote command is missing or unusable."""
31
+
32
+
33
+ class Cluster(ABC):
34
+ """Base class of the cluster-specific interfaces.
35
+
36
+ A cluster declares the remote commands needed for every section of
37
+ information (compute budgets, storage quotas and jobs) and parses their
38
+ output into the cluster-independent model of :mod:`hpcmon.models`. All the
39
+ commands of a query are executed through a single SSH session.
40
+
41
+ Parsers raise :class:`CollectionError` (or any other exception) when the
42
+ data of their section cannot be obtained; this is reported for that
43
+ section only, without affecting the others. Non-fatal problems can be
44
+ reported with :meth:`warn`.
45
+ """
46
+
47
+ #: Identifier of the cluster type, used for ``type`` in the configuration.
48
+ type_name: ClassVar[str] = ""
49
+ #: Human-readable name of the system.
50
+ display_name: ClassVar[str] = ""
51
+ #: Sections of information that this kind of cluster can provide.
52
+ supported_sections: ClassVar[Tuple[str, ...]] = SECTIONS
53
+ #: The GPUs of a GPU node, e.g. ``"4x NVIDIA A100 64 GB"`` (``None`` if
54
+ #: unknown); the ``gpus`` setting replaces it.
55
+ gpus: ClassVar[Optional[str]] = None
56
+ #: Estimated GPU-hours bought by one unit of a budget, by the
57
+ #: ``(resource, unit)`` of the budget (the resource is compared without
58
+ #: case, the unit exactly). Budgets missing here (CPU-only partitions,
59
+ #: storage billing, unknown units) get no estimate. The rates assume that
60
+ #: jobs use all the GPUs of the nodes they are charged for: on shared
61
+ #: nodes, no more than the share of cores and memory of their GPUs.
62
+ gpu_rates: ClassVar[Mapping[Tuple[str, str], float]] = {}
63
+
64
+ def __init__(self, config: ClusterConfig, runner: Optional[SSHRunner] = None) -> None:
65
+ self.config = config
66
+ self.runner = runner or SSHRunner(config.ssh, timeout=config.timeout, login_shell=config.login_shell)
67
+ self._commands: Dict[str, str] = {}
68
+ self._warnings: List[str] = []
69
+
70
+ @property
71
+ def name(self) -> str:
72
+ """Name of the cluster in the configuration."""
73
+ return self.config.name
74
+
75
+ @property
76
+ def gpu_label(self) -> Optional[str]:
77
+ """Description of the GPUs of a node: the setting, else the built-in one."""
78
+ label = self.config.gpus if self.config.gpus is not None else self.gpus
79
+ return (label or "").strip() or None
80
+
81
+ def gpu_rate(self, budget: ComputeBudget) -> Optional[float]:
82
+ """GPU-hours bought by one unit of ``budget`` (``None``: not convertible).
83
+
84
+ A rate in the ``gpu_hours_per_unit`` setting takes precedence (0
85
+ disables the estimate); otherwise :attr:`gpu_rates` is used.
86
+ """
87
+ configured = self.config.gpu_rate_for(budget.resource)
88
+ if configured is not None:
89
+ return configured or None
90
+ for (resource, unit), rate in self.gpu_rates.items():
91
+ if resource.lower() == budget.resource.lower() and unit == budget.unit:
92
+ return rate
93
+ return None
94
+
95
+ # -- interface to implement -------------------------------------------
96
+
97
+ @abstractmethod
98
+ def compute_commands(self) -> Dict[str, str]:
99
+ """Remote commands (by unique key) needed by :meth:`parse_compute`."""
100
+
101
+ @abstractmethod
102
+ def storage_commands(self) -> Dict[str, str]:
103
+ """Remote commands (by unique key) needed by :meth:`parse_storage`."""
104
+
105
+ @abstractmethod
106
+ def jobs_commands(self) -> Dict[str, str]:
107
+ """Remote commands (by unique key) needed by :meth:`parse_jobs`."""
108
+
109
+ @abstractmethod
110
+ def parse_compute(self, results: Results, today: dt.date) -> List[ComputeBudget]:
111
+ """Extract the compute budgets of the user's projects."""
112
+
113
+ @abstractmethod
114
+ def parse_storage(self, results: Results, today: dt.date) -> List[StorageQuota]:
115
+ """Extract the storage quotas of the user and their projects."""
116
+
117
+ @abstractmethod
118
+ def parse_jobs(self, results: Results, today: dt.date) -> List[Job]:
119
+ """Extract the jobs of the user."""
120
+
121
+ # -- helpers for implementations --------------------------------------
122
+
123
+ def output(self, results: Results, key: str, *, check: bool = True) -> str:
124
+ """Return the standard output of the command ``key``.
125
+
126
+ :param check: If true, a non-zero exit status is an error.
127
+ :raises CollectionError: if the command did not run, timed out or
128
+ (with ``check``) failed.
129
+ """
130
+ command = self._commands.get(key, key)
131
+ result = results.get(key)
132
+ if result is None:
133
+ raise CollectionError(f"no output from `{command}`")
134
+ if result.returncode is None:
135
+ raise CollectionError(f"`{command}` did not complete: {result.reason or 'no exit status'}")
136
+ if result.returncode == TIMEOUT_EXIT_CODE:
137
+ raise CollectionError(f"`{command}` timed out (time limit of {self.config.timeout:g} s reached)")
138
+ if check and result.returncode != 0:
139
+ raise CollectionError(f"`{command}` failed (exit status {result.returncode}){_detail(result)}")
140
+ return result.stdout
141
+
142
+ def warn(self, message: str) -> None:
143
+ """Report a non-fatal problem in the current collection."""
144
+ self._warnings.append(message)
145
+
146
+ # -- collection -------------------------------------------------------
147
+
148
+ def commands(self, sections: Iterable[str] = SECTIONS) -> Dict[str, str]:
149
+ """All the remote commands needed for ``sections``, in execution order."""
150
+ providers = {
151
+ "compute": self.compute_commands,
152
+ "storage": self.storage_commands,
153
+ "jobs": self.jobs_commands,
154
+ }
155
+ wanted = set(sections)
156
+ commands: Dict[str, str] = {}
157
+ for section in EXECUTION_ORDER:
158
+ if section in wanted:
159
+ for key, command in providers[section]().items():
160
+ commands.setdefault(key, command)
161
+ return commands
162
+
163
+ def collect(
164
+ self,
165
+ sections: Sequence[str] = SECTIONS,
166
+ today: Optional[dt.date] = None,
167
+ now: Optional[dt.datetime] = None,
168
+ ) -> ClusterReport:
169
+ """Query the cluster and return everything found for ``sections``.
170
+
171
+ This never raises for problems with the cluster: they are recorded in
172
+ the ``errors`` of the returned report instead.
173
+ """
174
+ unknown = [section for section in sections if section not in SECTIONS]
175
+ if unknown:
176
+ raise ValueError(f"unknown section(s): {', '.join(unknown)}")
177
+ sections = [section for section in SECTIONS if section in sections and section in self.supported_sections]
178
+ now = now or dt.datetime.now().astimezone()
179
+ today = today or now.date()
180
+ report = ClusterReport(
181
+ name=self.name, system=self.display_name, gpus=self.gpu_label, sections=sections, collected_at=now
182
+ )
183
+ self._commands = self.commands(sections)
184
+ self._warnings = []
185
+ if self.config.gpu_hours_per_unit and "compute" not in self.supported_sections:
186
+ self.warn("the gpu_hours_per_unit setting is ignored: this type of cluster reports no compute budgets")
187
+ try:
188
+ results = self.runner.run(self._commands)
189
+ except RemoteError as exc:
190
+ report.errors.append(str(exc))
191
+ report.failed_sections = list(sections)
192
+ return report
193
+ for key, result in results.items():
194
+ logger.debug(
195
+ "%s: `%s` exited with %s\n--- stdout ---\n%s\n--- stderr ---\n%s",
196
+ self.name, self._commands[key], result.returncode, result.stdout, result.stderr,
197
+ )
198
+
199
+ # Names of the projects known to the cluster, to detect typos in the
200
+ # "projects" setting (jobs are not used: the queue may be empty).
201
+ seen: Set[str] = set()
202
+ if "compute" in sections:
203
+ budgets = self._guard(report, "compute", self.parse_compute, results, today) or []
204
+ seen.update(name for budget in budgets for name in (budget.project, budget.account) if name)
205
+ report.budgets = [self._complete(budget) for budget in budgets if self._wants_budget(budget)]
206
+ if "compute" not in report.failed_sections:
207
+ self._check_gpu_rates(budgets)
208
+ if "storage" in sections:
209
+ quotas = self._guard(report, "storage", self.parse_storage, results, today) or []
210
+ seen.update(quota.project for quota in quotas if quota.project)
211
+ report.storage = [quota for quota in quotas if self.config.wants_project(quota.project)]
212
+ if "jobs" in sections:
213
+ jobs = self._guard(report, "jobs", self.parse_jobs, results, today)
214
+ if jobs is not None:
215
+ report.jobs = [job for job in jobs if self.config.wants_project(job.account or None)]
216
+ hidden = len(jobs) - len(report.jobs)
217
+ if hidden:
218
+ self.warn(f"{hidden} job(s) of other accounts hidden by the 'projects' setting")
219
+ checked = {"compute", "storage"} & set(sections)
220
+ if self.config.projects is not None and checked and not report.failed_sections:
221
+ known = {name.lower() for name in seen}
222
+ missing = [name for name in self.config.projects if name.lower() not in known]
223
+ if missing:
224
+ self.warn(f"project(s) not found on this cluster: {', '.join(missing)}")
225
+ report.warnings.extend(self._warnings)
226
+ return report
227
+
228
+ def _check_gpu_rates(self, budgets: List[ComputeBudget]) -> None:
229
+ """Warn about rates of the ``gpu_hours_per_unit`` setting that apply to
230
+ no budget of the cluster (e.g. typos in the resource names)."""
231
+ resources = sorted({budget.resource for budget in budgets})
232
+ unused = [name for name in self.config.gpu_hours_per_unit if name.lower() not in {r.lower() for r in resources}]
233
+ if unused:
234
+ known = f" (budget resources: {', '.join(resources)})" if resources else ""
235
+ self.warn(f"gpu_hours_per_unit: no budget of resource {', '.join(unused)} on this cluster{known}")
236
+
237
+ def _wants_budget(self, budget: ComputeBudget) -> bool:
238
+ if self.config.wants_project(budget.project):
239
+ return True
240
+ return budget.account is not None and self.config.wants_project(budget.account)
241
+
242
+ def _guard(
243
+ self,
244
+ report: ClusterReport,
245
+ section: str,
246
+ parser: Callable[[Results, dt.date], List[T]],
247
+ results: Results,
248
+ today: dt.date,
249
+ ) -> Optional[List[T]]:
250
+ try:
251
+ return parser(results, today)
252
+ except CollectionError as exc:
253
+ report.errors.append(f"{section}: {exc}")
254
+ except Exception as exc: # noqa: BLE001 - one broken parser must not hide the rest
255
+ logger.debug("%s: cannot parse %s", self.name, section, exc_info=True)
256
+ report.errors.append(f"{section}: unexpected output ({type(exc).__name__}: {exc})")
257
+ report.failed_sections.append(section)
258
+ return None
259
+
260
+ def _complete(self, budget: ComputeBudget) -> ComputeBudget:
261
+ if budget.end is None:
262
+ budget.end = self.config.end_date_for(budget.project)
263
+ # A rate set by the parser is kept, unless the configuration has one.
264
+ if budget.gpu_hours_per_unit is None or self.config.gpu_rate_for(budget.resource) is not None:
265
+ budget.gpu_hours_per_unit = self.gpu_rate(budget)
266
+ return budget
267
+
268
+
269
+ def _detail(result: CommandResult) -> str:
270
+ line = last_line(result.stderr, result.stdout)
271
+ return f": {line}" if line else ""