gpu-stack-inspector 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gpu_stack_inspector/__init__.py +8 -0
- gpu_stack_inspector/__main__.py +6 -0
- gpu_stack_inspector/cli.py +335 -0
- gpu_stack_inspector/collectors/__init__.py +1 -0
- gpu_stack_inspector/collectors/environment.py +31 -0
- gpu_stack_inspector/collectors/host.py +48 -0
- gpu_stack_inspector/collectors/nvidia.py +166 -0
- gpu_stack_inspector/collectors/python_env.py +63 -0
- gpu_stack_inspector/collectors/toolkit.py +160 -0
- gpu_stack_inspector/compatibility/__init__.py +1 -0
- gpu_stack_inspector/compatibility/data.json +24 -0
- gpu_stack_inspector/compatibility/loader.py +75 -0
- gpu_stack_inspector/compatibility/versions.py +25 -0
- gpu_stack_inspector/exit_codes.py +22 -0
- gpu_stack_inspector/models.py +349 -0
- gpu_stack_inspector/orchestrator.py +132 -0
- gpu_stack_inspector/privacy.py +237 -0
- gpu_stack_inspector/probes/__init__.py +1 -0
- gpu_stack_inspector/probes/metadata_probe.py +230 -0
- gpu_stack_inspector/probes/metadata_program.py +232 -0
- gpu_stack_inspector/probes/runtime_probe.py +153 -0
- gpu_stack_inspector/probes/runtime_program.py +101 -0
- gpu_stack_inspector/reports/__init__.py +1 -0
- gpu_stack_inspector/reports/json_report.py +43 -0
- gpu_stack_inspector/reports/markdown.py +182 -0
- gpu_stack_inspector/reports/terminal.py +298 -0
- gpu_stack_inspector/rules/__init__.py +1 -0
- gpu_stack_inspector/rules/catalog.py +21 -0
- gpu_stack_inspector/rules/comparison.py +73 -0
- gpu_stack_inspector/rules/compatibility.py +256 -0
- gpu_stack_inspector/rules/differences.py +64 -0
- gpu_stack_inspector/rules/engine.py +101 -0
- gpu_stack_inspector/rules/host.py +259 -0
- gpu_stack_inspector/rules/interpreters.py +422 -0
- gpu_stack_inspector/rules/torch_build.py +227 -0
- gpu_stack_inspector/runner.py +273 -0
- gpu_stack_inspector-0.1.0.dist-info/METADATA +134 -0
- gpu_stack_inspector-0.1.0.dist-info/RECORD +41 -0
- gpu_stack_inspector-0.1.0.dist-info/WHEEL +4 -0
- gpu_stack_inspector-0.1.0.dist-info/entry_points.txt +3 -0
- gpu_stack_inspector-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""GPU Stack Inspector: explain GPU differences between Python environments."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("gpu-stack-inspector")
|
|
7
|
+
except PackageNotFoundError: # imported from a source tree that was never installed
|
|
8
|
+
__version__ = "0+unknown"
|
|
@@ -0,0 +1,335 @@
|
|
|
1
|
+
"""Command-line interface: parses arguments and wires the layers together.
|
|
2
|
+
|
|
3
|
+
No diagnosis logic lives here; commands call collectors, rules and renderers.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import os
|
|
9
|
+
import sys
|
|
10
|
+
from enum import StrEnum
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Annotated
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
from rich.console import Console
|
|
16
|
+
from rich.table import Table
|
|
17
|
+
from rich.text import Text
|
|
18
|
+
|
|
19
|
+
from gpu_stack_inspector import __version__, orchestrator
|
|
20
|
+
from gpu_stack_inspector.exit_codes import ExitCode
|
|
21
|
+
from gpu_stack_inspector.models import FailureKind, Snapshot
|
|
22
|
+
from gpu_stack_inspector.privacy import RedactionContext, Redactor, redact_snapshot
|
|
23
|
+
from gpu_stack_inspector.reports.json_report import render_json
|
|
24
|
+
from gpu_stack_inspector.reports.markdown import render_markdown
|
|
25
|
+
from gpu_stack_inspector.reports.terminal import redaction_line, render_compare, render_terminal
|
|
26
|
+
from gpu_stack_inspector.rules.catalog import builtin_rules
|
|
27
|
+
from gpu_stack_inspector.rules.engine import RuleContext
|
|
28
|
+
from gpu_stack_inspector.runner import SubprocessRunner
|
|
29
|
+
|
|
30
|
+
HELP = (
|
|
31
|
+
"Explain why PyTorch can use the GPU in one Python environment but not another.\n\n"
|
|
32
|
+
"Inspection is read-only: nothing is installed or changed, no root is needed and no "
|
|
33
|
+
"network connection is made. PyTorch is only imported with `probe --runtime`."
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
app = typer.Typer(
|
|
37
|
+
name="gpu-stack-inspector",
|
|
38
|
+
help=HELP,
|
|
39
|
+
no_args_is_help=True,
|
|
40
|
+
add_completion=False,
|
|
41
|
+
# Rich tracebacks print local variables, which can hold environment values.
|
|
42
|
+
pretty_exceptions_enable=False,
|
|
43
|
+
context_settings={"help_option_names": ["-h", "--help"]},
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _show_version(value: bool) -> None:
|
|
48
|
+
if value:
|
|
49
|
+
typer.echo(f"gpu-stack-inspector {__version__}")
|
|
50
|
+
raise typer.Exit()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@app.callback()
|
|
54
|
+
def cli(
|
|
55
|
+
version: Annotated[
|
|
56
|
+
bool,
|
|
57
|
+
typer.Option(
|
|
58
|
+
"--version", callback=_show_version, is_eager=True, help="Show the version and exit."
|
|
59
|
+
),
|
|
60
|
+
] = False,
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Global options (the help text comes from HELP)."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
rules_app = typer.Typer(
|
|
66
|
+
help="List the checks and explain what each one looks at.", no_args_is_help=True
|
|
67
|
+
)
|
|
68
|
+
app.add_typer(rules_app, name="rules")
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@rules_app.command("list")
|
|
72
|
+
def rules_list() -> None:
|
|
73
|
+
"""List every check with its code."""
|
|
74
|
+
table = Table(box=None, pad_edge=False, show_edge=False)
|
|
75
|
+
table.add_column("Code", no_wrap=True)
|
|
76
|
+
table.add_column("Check")
|
|
77
|
+
for rule in builtin_rules():
|
|
78
|
+
table.add_row(Text(rule.id), Text(rule.title))
|
|
79
|
+
Console().print(table)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
@rules_app.command("explain")
|
|
83
|
+
def rules_explain(
|
|
84
|
+
code: Annotated[
|
|
85
|
+
str, typer.Argument(metavar="CODE", help="A finding code, e.g. GSI-TORCH-002.")
|
|
86
|
+
],
|
|
87
|
+
) -> None:
|
|
88
|
+
"""Explain what a check looks at, how it decides, and its sources."""
|
|
89
|
+
rule = builtin_rules().get(code.strip())
|
|
90
|
+
if rule is None:
|
|
91
|
+
typer.echo(
|
|
92
|
+
f"Unknown code {code}; `gpu-stack-inspector rules list` shows them all.", err=True
|
|
93
|
+
)
|
|
94
|
+
raise typer.Exit(ExitCode.ERROR)
|
|
95
|
+
console = Console()
|
|
96
|
+
console.print(Text(f"{rule.id} {rule.title}", style="bold"))
|
|
97
|
+
console.print()
|
|
98
|
+
console.print(Text(rule.explanation))
|
|
99
|
+
if rule.sources:
|
|
100
|
+
console.print()
|
|
101
|
+
console.print(Text("Sources:", style="bold"))
|
|
102
|
+
for source in rule.sources:
|
|
103
|
+
console.print(Text(f" {source}"))
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class OutputFormat(StrEnum):
|
|
107
|
+
TEXT = "text"
|
|
108
|
+
JSON = "json"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@app.command()
|
|
112
|
+
def inspect(
|
|
113
|
+
python: Annotated[
|
|
114
|
+
list[str] | None,
|
|
115
|
+
typer.Option(
|
|
116
|
+
"--python",
|
|
117
|
+
"-p",
|
|
118
|
+
metavar="PATH",
|
|
119
|
+
help=(
|
|
120
|
+
"Python interpreter to inspect. Repeat it to inspect several. "
|
|
121
|
+
"Default: the python on PATH."
|
|
122
|
+
),
|
|
123
|
+
),
|
|
124
|
+
] = None,
|
|
125
|
+
output_format: Annotated[
|
|
126
|
+
OutputFormat, typer.Option("--format", "-f", help="Output format.")
|
|
127
|
+
] = OutputFormat.TEXT,
|
|
128
|
+
) -> None:
|
|
129
|
+
"""Inspect this machine and Python interpreters, read-only.
|
|
130
|
+
|
|
131
|
+
Interpreters are inspected from their metadata only; nothing is imported into
|
|
132
|
+
them beyond the standard library.
|
|
133
|
+
"""
|
|
134
|
+
snapshot = _collect(python, compare=False)
|
|
135
|
+
redacted = redact_snapshot(snapshot)
|
|
136
|
+
if output_format is OutputFormat.JSON:
|
|
137
|
+
typer.echo(render_json(redacted), nl=False)
|
|
138
|
+
else:
|
|
139
|
+
render_terminal(redacted, Console())
|
|
140
|
+
if _unusable_interpreters(snapshot):
|
|
141
|
+
raise typer.Exit(ExitCode.CONTEXT_UNAVAILABLE)
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
@app.command()
|
|
145
|
+
def compare(
|
|
146
|
+
python: Annotated[
|
|
147
|
+
list[str],
|
|
148
|
+
typer.Option(
|
|
149
|
+
"--python",
|
|
150
|
+
"-p",
|
|
151
|
+
metavar="PATH",
|
|
152
|
+
help="An interpreter to compare. Give it at least twice.",
|
|
153
|
+
),
|
|
154
|
+
],
|
|
155
|
+
output_format: Annotated[
|
|
156
|
+
OutputFormat, typer.Option("--format", "-f", help="Output format.")
|
|
157
|
+
] = OutputFormat.TEXT,
|
|
158
|
+
show_all: Annotated[
|
|
159
|
+
bool, typer.Option("--all", help="Show the fields that are the same too.")
|
|
160
|
+
] = False,
|
|
161
|
+
) -> None:
|
|
162
|
+
"""Compare interpreters side by side: which fields differ, and what that means.
|
|
163
|
+
|
|
164
|
+
Read-only, like inspect: interpreters are inspected from their metadata only.
|
|
165
|
+
"""
|
|
166
|
+
if len(python) < 2:
|
|
167
|
+
raise typer.BadParameter("give --python at least twice", param_hint="--python")
|
|
168
|
+
snapshot = _collect(python, compare=True)
|
|
169
|
+
redacted = redact_snapshot(snapshot)
|
|
170
|
+
if output_format is OutputFormat.JSON:
|
|
171
|
+
typer.echo(render_json(redacted), nl=False)
|
|
172
|
+
else:
|
|
173
|
+
render_compare(redacted, Console(), show_all=show_all)
|
|
174
|
+
if _unusable_interpreters(snapshot):
|
|
175
|
+
raise typer.Exit(ExitCode.CONTEXT_UNAVAILABLE)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
@app.command()
|
|
179
|
+
def probe(
|
|
180
|
+
runtime: Annotated[
|
|
181
|
+
bool,
|
|
182
|
+
typer.Option(
|
|
183
|
+
"--runtime",
|
|
184
|
+
help="Agree to import torch in each interpreter and run a tiny CUDA operation.",
|
|
185
|
+
),
|
|
186
|
+
] = False,
|
|
187
|
+
python: Annotated[
|
|
188
|
+
list[str] | None,
|
|
189
|
+
typer.Option(
|
|
190
|
+
"--python",
|
|
191
|
+
"-p",
|
|
192
|
+
metavar="PATH",
|
|
193
|
+
help="Interpreter to test. Repeat it to test several. Default: the python on PATH.",
|
|
194
|
+
),
|
|
195
|
+
] = None,
|
|
196
|
+
output_format: Annotated[
|
|
197
|
+
OutputFormat, typer.Option("--format", "-f", help="Output format.")
|
|
198
|
+
] = OutputFormat.TEXT,
|
|
199
|
+
) -> None:
|
|
200
|
+
"""Prove CUDA works: import torch and run a tiny CUDA operation (needs --runtime).
|
|
201
|
+
|
|
202
|
+
This is the only command that imports packages of the inspected environments. It
|
|
203
|
+
creates a CUDA context and uses a few kilobytes of GPU memory for a moment, so it only
|
|
204
|
+
runs when you pass --runtime.
|
|
205
|
+
"""
|
|
206
|
+
if not runtime:
|
|
207
|
+
typer.echo(
|
|
208
|
+
"probe imports torch in each interpreter and runs a tiny CUDA operation, which "
|
|
209
|
+
"uses a little GPU memory. Run it again with --runtime to agree.",
|
|
210
|
+
err=True,
|
|
211
|
+
)
|
|
212
|
+
raise typer.Exit(ExitCode.ERROR)
|
|
213
|
+
count = len(python) if python else 1
|
|
214
|
+
typer.echo(
|
|
215
|
+
f"Importing torch and running a tiny CUDA operation in {count} interpreter(s)…", err=True
|
|
216
|
+
)
|
|
217
|
+
snapshot = _collect(python, compare=count >= 2, runtime=True)
|
|
218
|
+
redacted = redact_snapshot(snapshot)
|
|
219
|
+
if output_format is OutputFormat.JSON:
|
|
220
|
+
typer.echo(render_json(redacted), nl=False)
|
|
221
|
+
else:
|
|
222
|
+
render_terminal(redacted, Console())
|
|
223
|
+
if _unusable_interpreters(snapshot):
|
|
224
|
+
raise typer.Exit(ExitCode.CONTEXT_UNAVAILABLE)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class ReportFormat(StrEnum):
|
|
228
|
+
MARKDOWN = "markdown"
|
|
229
|
+
JSON = "json"
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@app.command()
|
|
233
|
+
def report(
|
|
234
|
+
python: Annotated[
|
|
235
|
+
list[str] | None,
|
|
236
|
+
typer.Option(
|
|
237
|
+
"--python",
|
|
238
|
+
"-p",
|
|
239
|
+
metavar="PATH",
|
|
240
|
+
help=(
|
|
241
|
+
"Interpreter to include. Repeat it to compare several. Default: the python on PATH."
|
|
242
|
+
),
|
|
243
|
+
),
|
|
244
|
+
] = None,
|
|
245
|
+
report_format: Annotated[
|
|
246
|
+
ReportFormat, typer.Option("--format", "-f", help="Report format.")
|
|
247
|
+
] = ReportFormat.MARKDOWN,
|
|
248
|
+
output: Annotated[
|
|
249
|
+
Path | None,
|
|
250
|
+
typer.Option(
|
|
251
|
+
"--output",
|
|
252
|
+
"-o",
|
|
253
|
+
metavar="FILE",
|
|
254
|
+
help="Write the report to FILE instead of printing it ('-' prints).",
|
|
255
|
+
),
|
|
256
|
+
] = None,
|
|
257
|
+
force: Annotated[bool, typer.Option("--force", help="Overwrite FILE if it exists.")] = False,
|
|
258
|
+
) -> None:
|
|
259
|
+
"""Write a redacted report to paste into a bug tracker.
|
|
260
|
+
|
|
261
|
+
By default the report is printed, so it can be read before it is shared. With two or
|
|
262
|
+
more interpreters it includes their comparison. The file named with --output is the
|
|
263
|
+
only thing this tool ever writes.
|
|
264
|
+
"""
|
|
265
|
+
to_file = output is not None and str(output) != "-"
|
|
266
|
+
if to_file and output is not None and output.exists() and not force:
|
|
267
|
+
typer.echo(f"{output} exists; use --force to overwrite it.", err=True)
|
|
268
|
+
raise typer.Exit(ExitCode.ERROR)
|
|
269
|
+
snapshot = _collect(python, compare=len(python or []) >= 2)
|
|
270
|
+
redacted = redact_snapshot(snapshot)
|
|
271
|
+
if report_format is ReportFormat.JSON:
|
|
272
|
+
rendered = render_json(redacted)
|
|
273
|
+
else:
|
|
274
|
+
rendered = render_markdown(redacted)
|
|
275
|
+
if to_file and output is not None:
|
|
276
|
+
output.write_text(rendered, encoding="utf-8")
|
|
277
|
+
note = redaction_line(redacted) or "Nothing needed redacting"
|
|
278
|
+
typer.echo(f"Wrote {output}. {note}. Read it before you share it.", err=True)
|
|
279
|
+
else:
|
|
280
|
+
typer.echo(rendered, nl=False)
|
|
281
|
+
if _unusable_interpreters(snapshot):
|
|
282
|
+
raise typer.Exit(ExitCode.CONTEXT_UNAVAILABLE)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _collect(python: list[str] | None, *, compare: bool, runtime: bool = False) -> Snapshot:
|
|
286
|
+
"""Inspect, run every check and, if asked, compare: the shared part of each command.
|
|
287
|
+
|
|
288
|
+
Without interpreters the python on PATH is used; the shell's python also goes into the
|
|
289
|
+
rule context. ``runtime`` is the user's consent to import torch and use CUDA.
|
|
290
|
+
"""
|
|
291
|
+
shell_python = next(iter(orchestrator.default_interpreters()), None)
|
|
292
|
+
pythons = python or ([shell_python] if shell_python else [])
|
|
293
|
+
if not pythons:
|
|
294
|
+
typer.echo("No python found on PATH; name interpreters with --python.", err=True)
|
|
295
|
+
snapshot = orchestrator.diagnose(
|
|
296
|
+
orchestrator.inspect(SubprocessRunner(), pythons, runtime=runtime),
|
|
297
|
+
RuleContext(shell_python=shell_python),
|
|
298
|
+
)
|
|
299
|
+
return orchestrator.with_comparison(snapshot) if compare else snapshot
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
_UNUSABLE = frozenset({FailureKind.NOT_FOUND, FailureKind.PERMISSION_DENIED})
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _unusable_interpreters(snapshot: Snapshot) -> bool:
|
|
306
|
+
"""A named interpreter that could not even be started: exit code 3, after the report.
|
|
307
|
+
|
|
308
|
+
An interpreter that starts and then crashes is a finding about that environment,
|
|
309
|
+
not missing context, so it does not count.
|
|
310
|
+
"""
|
|
311
|
+
return any(
|
|
312
|
+
interpreter.version is None and any(f.kind in _UNUSABLE for f in interpreter.failures)
|
|
313
|
+
for interpreter in snapshot.interpreters
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def main() -> None:
|
|
318
|
+
"""Console entry point.
|
|
319
|
+
|
|
320
|
+
Typer reports usage errors itself (exit code 2) and turns Ctrl+C into 130. Anything
|
|
321
|
+
else that escapes is a bug: print one redacted line instead of a traceback, which
|
|
322
|
+
could contain local values, and exit with 2. ``GSI_DEBUG=1`` shows the traceback.
|
|
323
|
+
"""
|
|
324
|
+
try:
|
|
325
|
+
app()
|
|
326
|
+
except Exception as exc: # SystemExit (normal exits) is not an Exception
|
|
327
|
+
if os.environ.get("GSI_DEBUG"):
|
|
328
|
+
raise
|
|
329
|
+
message = Redactor(RedactionContext.current()).text(str(exc)) or type(exc).__name__
|
|
330
|
+
typer.echo(
|
|
331
|
+
f"gpu-stack-inspector: internal error: {message}\n"
|
|
332
|
+
"Run it again with GSI_DEBUG=1 to see the traceback, and please report it.",
|
|
333
|
+
err=True,
|
|
334
|
+
)
|
|
335
|
+
sys.exit(ExitCode.ERROR)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Collectors read facts from the system; they never decide what a fact means."""
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Allowlisted CUDA-related environment variables of the tool's own process.
|
|
2
|
+
|
|
3
|
+
Target interpreters inherit them, so they explain differences such as a GPU hidden by
|
|
4
|
+
``CUDA_VISIBLE_DEVICES``. Nothing outside the allowlist is ever read: the rest of the
|
|
5
|
+
environment can hold tokens and passwords.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
from collections.abc import Mapping
|
|
12
|
+
|
|
13
|
+
from gpu_stack_inspector.models import EnvironmentVariable
|
|
14
|
+
|
|
15
|
+
ALLOWLIST: tuple[str, ...] = (
|
|
16
|
+
"CUDA_VISIBLE_DEVICES",
|
|
17
|
+
"CUDA_DEVICE_ORDER",
|
|
18
|
+
"NVIDIA_VISIBLE_DEVICES",
|
|
19
|
+
"NVIDIA_DRIVER_CAPABILITIES",
|
|
20
|
+
"CUDA_HOME",
|
|
21
|
+
"CUDA_PATH",
|
|
22
|
+
"LD_LIBRARY_PATH",
|
|
23
|
+
"PYTORCH_CUDA_ALLOC_CONF",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def collect_cuda_environment(
|
|
28
|
+
environ: Mapping[str, str] = os.environ,
|
|
29
|
+
) -> tuple[EnvironmentVariable, ...]:
|
|
30
|
+
"""The allowlisted variables that are set, in allowlist order."""
|
|
31
|
+
return tuple(EnvironmentVariable(name, environ[name]) for name in ALLOWLIST if name in environ)
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Operating system facts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import platform
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
|
|
8
|
+
from gpu_stack_inspector.models import HostFacts
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def collect_host(
|
|
12
|
+
*,
|
|
13
|
+
system: Callable[[], str] = platform.system,
|
|
14
|
+
machine: Callable[[], str] = platform.machine,
|
|
15
|
+
release: Callable[[], str] = platform.release,
|
|
16
|
+
os_release: Callable[[], dict[str, str]] = platform.freedesktop_os_release,
|
|
17
|
+
) -> HostFacts:
|
|
18
|
+
"""Read the OS name, architecture, kernel release and Linux distribution.
|
|
19
|
+
|
|
20
|
+
Never runs a command. The distribution comes from ``/etc/os-release`` (or
|
|
21
|
+
``/usr/lib/os-release``); if neither can be read it stays unknown.
|
|
22
|
+
"""
|
|
23
|
+
name = system() or "unknown"
|
|
24
|
+
kernel = release() or None
|
|
25
|
+
return HostFacts(
|
|
26
|
+
system=name,
|
|
27
|
+
machine=machine() or "unknown",
|
|
28
|
+
kernel=kernel,
|
|
29
|
+
distribution=_distribution(os_release) if name == "Linux" else None,
|
|
30
|
+
wsl=_is_wsl(kernel),
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _distribution(os_release: Callable[[], dict[str, str]]) -> str | None:
|
|
35
|
+
try:
|
|
36
|
+
info = os_release()
|
|
37
|
+
except OSError:
|
|
38
|
+
return None
|
|
39
|
+
pretty = info.get("PRETTY_NAME", "").strip()
|
|
40
|
+
if pretty:
|
|
41
|
+
return pretty
|
|
42
|
+
fallback = " ".join(part for part in (info.get("NAME", ""), info.get("VERSION_ID", "")) if part)
|
|
43
|
+
return fallback or None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _is_wsl(kernel: str | None) -> bool:
|
|
47
|
+
# WSL kernels are built by Microsoft and say so: "5.15.167.4-microsoft-standard-WSL2".
|
|
48
|
+
return kernel is not None and "microsoft" in kernel.lower()
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"""NVIDIA driver and GPU facts from nvidia-smi."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import csv
|
|
6
|
+
import re
|
|
7
|
+
import shutil
|
|
8
|
+
from collections.abc import Callable, Sequence
|
|
9
|
+
|
|
10
|
+
from gpu_stack_inspector.models import FailureKind, GpuDevice, NvidiaFacts, ProbeFailure
|
|
11
|
+
from gpu_stack_inspector.runner import CommandResult, CommandRunner, Outcome, probe_failure
|
|
12
|
+
|
|
13
|
+
PROBE = "nvidia-smi"
|
|
14
|
+
TIMEOUT = 8.0
|
|
15
|
+
"""nvidia-smi answers in well under a second; a hung driver must not hang the tool."""
|
|
16
|
+
|
|
17
|
+
_ENV = {"LC_ALL": "C"} # decimal points and English messages, whatever the user's locale
|
|
18
|
+
_FIELDS = ("index", "name", "driver_version", "memory.total", "compute_cap")
|
|
19
|
+
_UNAVAILABLE = frozenset(
|
|
20
|
+
{"", "n/a", "[n/a]", "not supported", "[not supported]", "[unknown error]"}
|
|
21
|
+
)
|
|
22
|
+
_COMPUTE_CAPABILITY = re.compile(r"[0-9]+\.[0-9]+")
|
|
23
|
+
_TABLE_HEADER = re.compile(
|
|
24
|
+
r"Driver Version:\s*(?P<driver>[0-9][0-9.]*)\s+CUDA Version:\s*(?P<cuda>[0-9][0-9.]*)"
|
|
25
|
+
)
|
|
26
|
+
_VERSION_LINE = re.compile(r"^\s*(?P<key>[A-Za-z -]+?)\s*:\s*(?P<value>\S+)\s*$", re.MULTILINE)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def collect_nvidia(
|
|
30
|
+
runner: CommandRunner,
|
|
31
|
+
*,
|
|
32
|
+
which: Callable[[str], str | None] = shutil.which,
|
|
33
|
+
timeout: float = TIMEOUT,
|
|
34
|
+
) -> NvidiaFacts:
|
|
35
|
+
"""Ask nvidia-smi for the driver version and the GPUs it can see.
|
|
36
|
+
|
|
37
|
+
Only read-only queries are run. A missing nvidia-smi is reported as such; a failing
|
|
38
|
+
one is recorded as a failure and no further queries are attempted.
|
|
39
|
+
"""
|
|
40
|
+
smi = which("nvidia-smi")
|
|
41
|
+
if smi is None:
|
|
42
|
+
return NvidiaFacts(smi_path=None)
|
|
43
|
+
|
|
44
|
+
fields: Sequence[str] = _FIELDS
|
|
45
|
+
result = runner.run(_query_argv(smi, fields), timeout=timeout, env_overrides=_ENV)
|
|
46
|
+
if _rejects_field(result, "compute_cap"):
|
|
47
|
+
# Drivers older than about R510 do not know compute_cap; ask without it.
|
|
48
|
+
fields = tuple(field for field in _FIELDS if field != "compute_cap")
|
|
49
|
+
result = runner.run(_query_argv(smi, fields), timeout=timeout, env_overrides=_ENV)
|
|
50
|
+
if _no_devices(result):
|
|
51
|
+
# The driver answered but sees no GPU: zero devices is the fact to report.
|
|
52
|
+
versions = _versions(runner, smi, timeout)
|
|
53
|
+
return NvidiaFacts(
|
|
54
|
+
smi_path=smi,
|
|
55
|
+
driver_version=versions.get("driver version"),
|
|
56
|
+
max_cuda_version=versions.get("cuda version"),
|
|
57
|
+
)
|
|
58
|
+
failure = probe_failure(PROBE, result)
|
|
59
|
+
if failure is not None:
|
|
60
|
+
return NvidiaFacts(smi_path=smi, failures=(failure,))
|
|
61
|
+
|
|
62
|
+
gpus, driver_version, failures = _parse_query(result.stdout, fields)
|
|
63
|
+
versions = _versions(runner, smi, timeout)
|
|
64
|
+
if "cuda version" not in versions:
|
|
65
|
+
failures.append(_malformed("nvidia-smi did not report the driver's maximum CUDA version"))
|
|
66
|
+
return NvidiaFacts(
|
|
67
|
+
smi_path=smi,
|
|
68
|
+
driver_version=driver_version or versions.get("driver version"),
|
|
69
|
+
max_cuda_version=versions.get("cuda version"),
|
|
70
|
+
gpus=tuple(gpus),
|
|
71
|
+
failures=tuple(failures),
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _versions(runner: CommandRunner, smi: str, timeout: float) -> dict[str, str]:
|
|
76
|
+
"""Driver and maximum CUDA version from ``nvidia-smi --version``, keys lower-cased.
|
|
77
|
+
|
|
78
|
+
Its output looks like ``CUDA Version : 13.2``.
|
|
79
|
+
"""
|
|
80
|
+
result = runner.run([smi, "--version"], timeout=timeout, env_overrides=_ENV)
|
|
81
|
+
if result.ok:
|
|
82
|
+
return {
|
|
83
|
+
match["key"].strip().lower(): match["value"]
|
|
84
|
+
for match in _VERSION_LINE.finditer(result.stdout)
|
|
85
|
+
}
|
|
86
|
+
# Older drivers have no --version; the classic table's header has both values:
|
|
87
|
+
# "| NVIDIA-SMI 470.256.02 Driver Version: 470.256.02 CUDA Version: 11.4 |".
|
|
88
|
+
# Only the header is read; the process list further down is never parsed or kept.
|
|
89
|
+
table = runner.run([smi], timeout=timeout, env_overrides=_ENV)
|
|
90
|
+
header = _TABLE_HEADER.search(table.stdout) if table.ok else None
|
|
91
|
+
if header is None:
|
|
92
|
+
return {}
|
|
93
|
+
return {"driver version": header["driver"], "cuda version": header["cuda"]}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _rejects_field(result: CommandResult, field: str) -> bool:
|
|
97
|
+
# nvidia-smi prints this on stdout: Field "compute_cap" is not a valid field to query.
|
|
98
|
+
return result.outcome is Outcome.NONZERO and f'"{field}" is not a valid field' in result.stdout
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def _no_devices(result: CommandResult) -> bool:
|
|
102
|
+
return result.outcome is Outcome.NONZERO and "no devices were found" in result.stdout.lower()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _query_argv(smi: str, fields: Sequence[str]) -> list[str]:
|
|
106
|
+
return [smi, f"--query-gpu={','.join(fields)}", "--format=csv,noheader,nounits"]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _parse_query(
|
|
110
|
+
stdout: str, fields: Sequence[str]
|
|
111
|
+
) -> tuple[list[GpuDevice], str | None, list[ProbeFailure]]:
|
|
112
|
+
gpus: list[GpuDevice] = []
|
|
113
|
+
failures: list[ProbeFailure] = []
|
|
114
|
+
driver_version: str | None = None
|
|
115
|
+
try:
|
|
116
|
+
rows = list(csv.reader(stdout.splitlines(), skipinitialspace=True))
|
|
117
|
+
except csv.Error as exc:
|
|
118
|
+
return [], None, [_malformed(f"unreadable CSV output: {exc}")]
|
|
119
|
+
for number, row in enumerate(rows, start=1):
|
|
120
|
+
if not any(cell.strip() for cell in row):
|
|
121
|
+
continue
|
|
122
|
+
if len(row) != len(fields):
|
|
123
|
+
failures.append(
|
|
124
|
+
_malformed(f"line {number} has {len(row)} fields, expected {len(fields)}")
|
|
125
|
+
)
|
|
126
|
+
continue
|
|
127
|
+
values = dict(zip(fields, (cell.strip() for cell in row), strict=True))
|
|
128
|
+
index = _integer(values["index"])
|
|
129
|
+
name = _value(values["name"])
|
|
130
|
+
if index is None or name is None:
|
|
131
|
+
failures.append(_malformed(f"line {number} has no GPU index or name"))
|
|
132
|
+
continue
|
|
133
|
+
gpus.append(
|
|
134
|
+
GpuDevice(
|
|
135
|
+
index=index,
|
|
136
|
+
name=name,
|
|
137
|
+
memory_total_mib=_integer(values.get("memory.total", "")),
|
|
138
|
+
compute_capability=_compute_capability(values.get("compute_cap", "")),
|
|
139
|
+
)
|
|
140
|
+
)
|
|
141
|
+
driver_version = driver_version or _value(values["driver_version"])
|
|
142
|
+
return gpus, driver_version, failures
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _malformed(detail: str) -> ProbeFailure:
|
|
146
|
+
return ProbeFailure(PROBE, FailureKind.MALFORMED_OUTPUT, detail)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _value(raw: str) -> str | None:
|
|
150
|
+
value = raw.strip()
|
|
151
|
+
return None if value.lower() in _UNAVAILABLE else value
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def _integer(raw: str) -> int | None:
|
|
155
|
+
value = _value(raw)
|
|
156
|
+
if value is None:
|
|
157
|
+
return None
|
|
158
|
+
try:
|
|
159
|
+
return int(value)
|
|
160
|
+
except ValueError:
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def _compute_capability(raw: str) -> str | None:
|
|
165
|
+
value = _value(raw)
|
|
166
|
+
return value if value is not None and _COMPUTE_CAPABILITY.fullmatch(value) else None
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""Target Python interpreters, inspected from the outside.
|
|
2
|
+
|
|
3
|
+
The tool never imports a target environment's packages into its own process; it runs
|
|
4
|
+
the target interpreter as a subprocess.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import os
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from gpu_stack_inspector.models import FailureKind, ProbeFailure
|
|
13
|
+
|
|
14
|
+
PROBE = "python"
|
|
15
|
+
_INTERPRETERS_IN_AN_ENVIRONMENT = (
|
|
16
|
+
Path("bin", "python"),
|
|
17
|
+
Path("bin", "python3"),
|
|
18
|
+
Path("Scripts", "python.exe"),
|
|
19
|
+
Path("python.exe"), # conda on Windows
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def interpreter_path(
|
|
24
|
+
raw: str, *, cwd: Path | None = None, windows: bool = os.name == "nt"
|
|
25
|
+
) -> tuple[Path | None, ProbeFailure | None]:
|
|
26
|
+
"""Turn a ``--python`` argument into an absolute path that can be run.
|
|
27
|
+
|
|
28
|
+
``~`` is expanded and relative paths are taken from ``cwd``. Symlinks are kept on
|
|
29
|
+
purpose: a virtual environment's ``bin/python`` is a symlink to the base
|
|
30
|
+
interpreter, and only the symlink's location tells Python it runs inside the venv.
|
|
31
|
+
On Windows a path without a suffix also matches ``<path>.exe``, as it does in the
|
|
32
|
+
shell (``.venv\\Scripts\\python``). Returns the path, or the reason it cannot be used.
|
|
33
|
+
"""
|
|
34
|
+
text = raw.strip()
|
|
35
|
+
if not text:
|
|
36
|
+
return None, _failure(FailureKind.NOT_FOUND, "no interpreter path was given")
|
|
37
|
+
path = Path(text).expanduser()
|
|
38
|
+
if not path.is_absolute():
|
|
39
|
+
path = (cwd or Path.cwd()) / path
|
|
40
|
+
if windows and not path.exists() and not path.suffix and path.with_suffix(".exe").exists():
|
|
41
|
+
path = path.with_suffix(".exe")
|
|
42
|
+
if not path.exists():
|
|
43
|
+
if path.is_symlink():
|
|
44
|
+
target = path.readlink()
|
|
45
|
+
detail = f"{path} is a symlink to {target}, which does not exist"
|
|
46
|
+
else:
|
|
47
|
+
detail = f"{path} does not exist"
|
|
48
|
+
return None, _failure(FailureKind.NOT_FOUND, detail)
|
|
49
|
+
if path.is_dir():
|
|
50
|
+
detail = f"{path} is a directory, not an interpreter"
|
|
51
|
+
inside = next(
|
|
52
|
+
(path / c for c in _INTERPRETERS_IN_AN_ENVIRONMENT if (path / c).is_file()), None
|
|
53
|
+
)
|
|
54
|
+
if inside is not None:
|
|
55
|
+
detail += f"; did you mean {inside}?"
|
|
56
|
+
return None, _failure(FailureKind.NOT_FOUND, detail)
|
|
57
|
+
if not os.access(path, os.X_OK):
|
|
58
|
+
return None, _failure(FailureKind.PERMISSION_DENIED, f"{path} is not executable")
|
|
59
|
+
return path, None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _failure(kind: FailureKind, detail: str) -> ProbeFailure:
|
|
63
|
+
return ProbeFailure(PROBE, kind, detail)
|