gpu-stack-inspector 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gpu_stack_inspector/__init__.py +8 -0
- gpu_stack_inspector/__main__.py +6 -0
- gpu_stack_inspector/cli.py +335 -0
- gpu_stack_inspector/collectors/__init__.py +1 -0
- gpu_stack_inspector/collectors/environment.py +31 -0
- gpu_stack_inspector/collectors/host.py +48 -0
- gpu_stack_inspector/collectors/nvidia.py +166 -0
- gpu_stack_inspector/collectors/python_env.py +63 -0
- gpu_stack_inspector/collectors/toolkit.py +160 -0
- gpu_stack_inspector/compatibility/__init__.py +1 -0
- gpu_stack_inspector/compatibility/data.json +24 -0
- gpu_stack_inspector/compatibility/loader.py +75 -0
- gpu_stack_inspector/compatibility/versions.py +25 -0
- gpu_stack_inspector/exit_codes.py +22 -0
- gpu_stack_inspector/models.py +349 -0
- gpu_stack_inspector/orchestrator.py +132 -0
- gpu_stack_inspector/privacy.py +237 -0
- gpu_stack_inspector/probes/__init__.py +1 -0
- gpu_stack_inspector/probes/metadata_probe.py +230 -0
- gpu_stack_inspector/probes/metadata_program.py +232 -0
- gpu_stack_inspector/probes/runtime_probe.py +153 -0
- gpu_stack_inspector/probes/runtime_program.py +101 -0
- gpu_stack_inspector/reports/__init__.py +1 -0
- gpu_stack_inspector/reports/json_report.py +43 -0
- gpu_stack_inspector/reports/markdown.py +182 -0
- gpu_stack_inspector/reports/terminal.py +298 -0
- gpu_stack_inspector/rules/__init__.py +1 -0
- gpu_stack_inspector/rules/catalog.py +21 -0
- gpu_stack_inspector/rules/comparison.py +73 -0
- gpu_stack_inspector/rules/compatibility.py +256 -0
- gpu_stack_inspector/rules/differences.py +64 -0
- gpu_stack_inspector/rules/engine.py +101 -0
- gpu_stack_inspector/rules/host.py +259 -0
- gpu_stack_inspector/rules/interpreters.py +422 -0
- gpu_stack_inspector/rules/torch_build.py +227 -0
- gpu_stack_inspector/runner.py +273 -0
- gpu_stack_inspector-0.1.0.dist-info/METADATA +134 -0
- gpu_stack_inspector-0.1.0.dist-info/RECORD +41 -0
- gpu_stack_inspector-0.1.0.dist-info/WHEEL +4 -0
- gpu_stack_inspector-0.1.0.dist-info/entry_points.txt +3 -0
- gpu_stack_inspector-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""Runs the collectors and assembles the snapshot; no diagnosis happens here."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
import string
|
|
8
|
+
from collections.abc import Callable, Mapping, Sequence
|
|
9
|
+
from dataclasses import replace
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from gpu_stack_inspector import __version__
|
|
13
|
+
from gpu_stack_inspector.collectors.environment import collect_cuda_environment
|
|
14
|
+
from gpu_stack_inspector.collectors.host import collect_host
|
|
15
|
+
from gpu_stack_inspector.collectors.nvidia import collect_nvidia
|
|
16
|
+
from gpu_stack_inspector.collectors.toolkit import DEFAULT_ROOTS, collect_toolkits
|
|
17
|
+
from gpu_stack_inspector.models import (
|
|
18
|
+
HostFacts,
|
|
19
|
+
InterpreterFacts,
|
|
20
|
+
ProbeFailure,
|
|
21
|
+
Snapshot,
|
|
22
|
+
utc_timestamp,
|
|
23
|
+
)
|
|
24
|
+
from gpu_stack_inspector.probes.metadata_probe import probe_interpreter
|
|
25
|
+
from gpu_stack_inspector.probes.runtime_probe import run_runtime_probe
|
|
26
|
+
from gpu_stack_inspector.rules.catalog import builtin_rules
|
|
27
|
+
from gpu_stack_inspector.rules.comparison import compare_interpreters
|
|
28
|
+
from gpu_stack_inspector.rules.engine import RuleContext, evaluate
|
|
29
|
+
from gpu_stack_inspector.runner import CommandRunner
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def inspect_host(
|
|
33
|
+
runner: CommandRunner,
|
|
34
|
+
*,
|
|
35
|
+
which: Callable[[str], str | None] = shutil.which,
|
|
36
|
+
environ: Mapping[str, str] = os.environ,
|
|
37
|
+
toolkit_roots: Sequence[Path] = DEFAULT_ROOTS,
|
|
38
|
+
host: Callable[[], HostFacts] = collect_host,
|
|
39
|
+
) -> Snapshot:
|
|
40
|
+
"""Collect the facts about this machine (not yet about any interpreter)."""
|
|
41
|
+
nvidia = collect_nvidia(runner, which=which)
|
|
42
|
+
toolkit = collect_toolkits(runner, which=which, roots=toolkit_roots, environ=environ)
|
|
43
|
+
return Snapshot(
|
|
44
|
+
tool_version=__version__,
|
|
45
|
+
created_at=utc_timestamp(),
|
|
46
|
+
host=host(),
|
|
47
|
+
nvidia=nvidia,
|
|
48
|
+
toolkit=toolkit,
|
|
49
|
+
cuda_environment=collect_cuda_environment(environ),
|
|
50
|
+
probe_failures=(*nvidia.failures, *toolkit.failures),
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def inspect(
|
|
55
|
+
runner: CommandRunner,
|
|
56
|
+
pythons: Sequence[str],
|
|
57
|
+
*,
|
|
58
|
+
cwd: Path | None = None,
|
|
59
|
+
which: Callable[[str], str | None] = shutil.which,
|
|
60
|
+
environ: Mapping[str, str] = os.environ,
|
|
61
|
+
toolkit_roots: Sequence[Path] = DEFAULT_ROOTS,
|
|
62
|
+
host: Callable[[], HostFacts] = collect_host,
|
|
63
|
+
runtime: bool = False,
|
|
64
|
+
) -> Snapshot:
|
|
65
|
+
"""Inspect the machine and each named interpreter, labelled A, B, C… in order.
|
|
66
|
+
|
|
67
|
+
Interpreter failures are added to the snapshot's list tagged with their label, so a
|
|
68
|
+
report can say which environment could not be checked. With ``runtime`` (the user's
|
|
69
|
+
explicit consent), torch is also imported and CUDA used once in every interpreter
|
|
70
|
+
that started and has torch installed.
|
|
71
|
+
"""
|
|
72
|
+
snapshot = inspect_host(
|
|
73
|
+
runner, which=which, environ=environ, toolkit_roots=toolkit_roots, host=host
|
|
74
|
+
)
|
|
75
|
+
interpreters = tuple(
|
|
76
|
+
probe_interpreter(runner, interpreter_label(index), raw, cwd=cwd)
|
|
77
|
+
for index, raw in enumerate(pythons)
|
|
78
|
+
)
|
|
79
|
+
if runtime:
|
|
80
|
+
interpreters = tuple(
|
|
81
|
+
replace(i, runtime=run_runtime_probe(runner, i.path))
|
|
82
|
+
if i.version is not None and i.torch is not None
|
|
83
|
+
else i
|
|
84
|
+
for i in interpreters
|
|
85
|
+
)
|
|
86
|
+
return replace(
|
|
87
|
+
snapshot,
|
|
88
|
+
interpreters=interpreters,
|
|
89
|
+
probe_failures=(*snapshot.probe_failures, *_tagged_failures(interpreters)),
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def interpreter_label(index: int) -> str:
|
|
94
|
+
"""A, B, … Z, then #27, #28 …"""
|
|
95
|
+
return string.ascii_uppercase[index] if index < 26 else f"#{index + 1}"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _tagged_failures(interpreters: Sequence[InterpreterFacts]) -> list[ProbeFailure]:
|
|
99
|
+
failures = [
|
|
100
|
+
replace(failure, probe=f"{failure.probe} {facts.label}")
|
|
101
|
+
for facts in interpreters
|
|
102
|
+
for failure in facts.failures
|
|
103
|
+
]
|
|
104
|
+
failures += [
|
|
105
|
+
replace(facts.runtime.failure, probe=f"{facts.runtime.failure.probe} {facts.label}")
|
|
106
|
+
for facts in interpreters
|
|
107
|
+
if facts.runtime is not None and facts.runtime.failure is not None
|
|
108
|
+
]
|
|
109
|
+
return failures
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def default_interpreters(which: Callable[[str], str | None] = shutil.which) -> list[str]:
|
|
113
|
+
"""The interpreter the user's shell would run: ``python``, else ``python3``, on PATH.
|
|
114
|
+
|
|
115
|
+
Not the tool's own interpreter: installed with pipx or ``uv tool`` it lives in an
|
|
116
|
+
isolated environment that tells the user nothing about their projects.
|
|
117
|
+
"""
|
|
118
|
+
for name in ("python", "python3"):
|
|
119
|
+
found = which(name)
|
|
120
|
+
if found:
|
|
121
|
+
return [found]
|
|
122
|
+
return []
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def diagnose(snapshot: Snapshot, context: RuleContext | None = None) -> Snapshot:
|
|
126
|
+
"""The snapshot with the findings of every built-in rule attached."""
|
|
127
|
+
return replace(snapshot, findings=evaluate(snapshot, builtin_rules(), context))
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def with_comparison(snapshot: Snapshot) -> Snapshot:
|
|
131
|
+
"""The snapshot with its interpreters compared field by field."""
|
|
132
|
+
return replace(snapshot, comparison=compare_interpreters(snapshot.interpreters))
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
"""Making collected text safe to print and to share.
|
|
2
|
+
|
|
3
|
+
Everything that leaves the tool (terminal, JSON, Markdown) is built from redacted data:
|
|
4
|
+
redaction runs on the snapshot *before* any renderer sees it.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import getpass
|
|
10
|
+
import re
|
|
11
|
+
import socket
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from dataclasses import dataclass, fields, is_dataclass, replace
|
|
15
|
+
from enum import Enum
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import TYPE_CHECKING, Any, TypeVar, cast
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from gpu_stack_inspector.models import Snapshot
|
|
21
|
+
|
|
22
|
+
T = TypeVar("T")
|
|
23
|
+
|
|
24
|
+
_ESCAPE_SEQUENCE = re.compile(
|
|
25
|
+
r"\x1b\[[0-?]*[ -/]*[@-~]" # CSI: colours, cursor movement, screen clearing
|
|
26
|
+
r"|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)?" # OSC: window titles, hyperlinks
|
|
27
|
+
r"|\x1b[@-_]" # other two-byte escapes
|
|
28
|
+
)
|
|
29
|
+
_CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def strip_control(text: str) -> str:
|
|
33
|
+
"""Remove terminal escape sequences and control characters, keeping tabs and newlines.
|
|
34
|
+
|
|
35
|
+
Collected output can come from a broken or hostile environment; without this a
|
|
36
|
+
crafted package name could rewrite the user's terminal when a report is shown.
|
|
37
|
+
"""
|
|
38
|
+
return _CONTROL.sub("", _ESCAPE_SEQUENCE.sub("", text))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
@dataclass(frozen=True, slots=True)
|
|
42
|
+
class RedactionContext:
|
|
43
|
+
"""The identifying values of the machine the report is made on."""
|
|
44
|
+
|
|
45
|
+
home: str | None = None
|
|
46
|
+
user: str | None = None
|
|
47
|
+
hostname: str | None = None
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def current(cls) -> RedactionContext:
|
|
51
|
+
"""Read the values for the current user; missing ones stay ``None``."""
|
|
52
|
+
return cls(
|
|
53
|
+
home=_safe(lambda: str(Path.home())),
|
|
54
|
+
user=_safe(getpass.getuser),
|
|
55
|
+
hostname=_safe(socket.gethostname),
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _safe(read: Callable[[], str]) -> str | None:
|
|
60
|
+
try:
|
|
61
|
+
value = read()
|
|
62
|
+
except (OSError, KeyError, RuntimeError):
|
|
63
|
+
return None
|
|
64
|
+
return value or None
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
Replacement = str | Callable[[re.Match[str]], str]
|
|
68
|
+
|
|
69
|
+
_REDACTED = "<redacted>"
|
|
70
|
+
_REDACTED_TOKEN = "<redacted-token>" # noqa: S105 - the placeholder, not a token
|
|
71
|
+
|
|
72
|
+
_URL_CREDENTIALS = re.compile(r"\b([a-zA-Z][a-zA-Z0-9+.-]*://)([^/\s?#]*)@")
|
|
73
|
+
"""``scheme://user:password@`` — greedy up to the last ``@`` of the authority."""
|
|
74
|
+
|
|
75
|
+
_NAME_AND_SEPARATOR = re.compile(r"""(?<![\w-])([A-Za-z_][A-Za-z0-9_]*)(["']?\s*[=:]\s*)""")
|
|
76
|
+
_VALUE = re.compile(r"""("[^"]*"|'[^']*'|[^\s,;}]+)""")
|
|
77
|
+
"""``NAME=value``, ``name: value``, ``"name": "value"``; the name decides if it is secret."""
|
|
78
|
+
|
|
79
|
+
_SECRET_SUFFIXES = ("token", "secret", "password", "passwd", "apikey", "credential", "credentials")
|
|
80
|
+
_SECRET_PARTS = frozenset({"key", "auth", "pass", "pat"})
|
|
81
|
+
|
|
82
|
+
_TOKENS = re.compile(
|
|
83
|
+
r"(?<![A-Za-z0-9_-])(?:"
|
|
84
|
+
r"gh[pousr]_[A-Za-z0-9]{36,}" # GitHub
|
|
85
|
+
r"|github_pat_[A-Za-z0-9_]{22,}"
|
|
86
|
+
r"|hf_[A-Za-z0-9]{30,}" # Hugging Face
|
|
87
|
+
r"|sk-[A-Za-z0-9_-]{20,}" # OpenAI, Anthropic
|
|
88
|
+
r"|(?:AKIA|ASIA)[0-9A-Z]{16}" # AWS access key id
|
|
89
|
+
r"|xox[abprs]-[A-Za-z0-9-]{10,}" # Slack
|
|
90
|
+
r"|eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}" # JWT
|
|
91
|
+
r")(?![A-Za-z0-9_-])"
|
|
92
|
+
)
|
|
93
|
+
_BEARER = re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{8,}")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _is_secret_name(name: str) -> bool:
|
|
97
|
+
"""``HF_TOKEN``, ``accessToken``, ``DB_PASSWORD``, ``OPENAI_API_KEY``: yes.
|
|
98
|
+
``tokenizers``, ``CUDA_HOME``, ``author``, ``KEYBOARD``: no."""
|
|
99
|
+
parts = [part for part in name.lower().split("_") if part]
|
|
100
|
+
return any(part in _SECRET_PARTS or part.endswith(_SECRET_SUFFIXES) for part in parts)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
_MIN_NAME_LENGTH = 3
|
|
104
|
+
_GENERIC_NAMES = frozenset(
|
|
105
|
+
{
|
|
106
|
+
"root", "user", "admin", "guest", "localhost", "linux", "ubuntu", "debian",
|
|
107
|
+
"fedora", "arch", "archlinux", "manjaro", "pop-os", "mint", "kali", "raspberrypi",
|
|
108
|
+
}
|
|
109
|
+
) # fmt: skip
|
|
110
|
+
"""Names that identify nobody and appear in ordinary text (distribution names as hosts)."""
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _word_pattern(name: str) -> re.Pattern[str] | None:
|
|
114
|
+
if len(name) < _MIN_NAME_LENGTH or name.lower() in _GENERIC_NAMES:
|
|
115
|
+
return None
|
|
116
|
+
# Whole words only: not inside "malice" or "alice_env", not an already redacted
|
|
117
|
+
# "<user>", but a sentence-ending period is fine.
|
|
118
|
+
return re.compile(rf"(?<![\w.<>-]){re.escape(name)}(?![\w<>-]|\.\w)")
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# Other accounts' home directories. The name may not start with "<", so already redacted
|
|
122
|
+
# paths are left alone.
|
|
123
|
+
_OTHER_HOMES: tuple[tuple[re.Pattern[str], str], ...] = (
|
|
124
|
+
(re.compile(r"(/home/)[^/\s:'\"<>]+"), r"\1<user>"),
|
|
125
|
+
(re.compile(r"(/Users/)[^/\s:'\"<>]+"), r"\1<user>"),
|
|
126
|
+
(re.compile(r"([A-Za-z]:\\Users\\)[^\\\s:'\"<>]+"), r"\1<user>"),
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class Redactor:
|
|
131
|
+
"""Removes identifying and secret values from text and from whole snapshots.
|
|
132
|
+
|
|
133
|
+
One redactor is used per report; :attr:`summary` then tells how many values of each
|
|
134
|
+
kind were removed, which the report shows so the user knows redaction happened.
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
def __init__(self, context: RedactionContext) -> None:
|
|
138
|
+
self._counts: Counter[str] = Counter()
|
|
139
|
+
self._steps: list[tuple[str, re.Pattern[str], Replacement]] = []
|
|
140
|
+
home = context.home.rstrip("/\\") if context.home else ""
|
|
141
|
+
if len(home) > 1: # a home of "/" would redact every absolute path
|
|
142
|
+
boundary = r"(?=[/\\]|$|[^\w.-])"
|
|
143
|
+
self._steps.append(("home_path", re.compile(re.escape(home) + boundary), "~"))
|
|
144
|
+
for pattern, replacement in _OTHER_HOMES:
|
|
145
|
+
self._steps.append(("home_path", pattern, replacement))
|
|
146
|
+
# Host before user: "alice-laptop" is a host name even when the user is "alice".
|
|
147
|
+
for category, name, placeholder in (
|
|
148
|
+
("host_name", context.hostname, "<host>"),
|
|
149
|
+
("user_name", context.user, "<user>"),
|
|
150
|
+
):
|
|
151
|
+
word = _word_pattern(name) if name else None
|
|
152
|
+
if word is not None:
|
|
153
|
+
self._steps.append((category, word, placeholder))
|
|
154
|
+
|
|
155
|
+
def text(self, value: str) -> str:
|
|
156
|
+
"""Return ``value`` without control characters, identifying or secret values."""
|
|
157
|
+
result = strip_control(value)
|
|
158
|
+
result = _URL_CREDENTIALS.sub(self._url_credentials, result)
|
|
159
|
+
result = self._secret_assignments(result)
|
|
160
|
+
for token_pattern, placeholder in (
|
|
161
|
+
(_TOKENS, _REDACTED_TOKEN),
|
|
162
|
+
(_BEARER, r"\1" + _REDACTED_TOKEN),
|
|
163
|
+
):
|
|
164
|
+
result, count = token_pattern.subn(placeholder, result)
|
|
165
|
+
self._counts["token"] += count
|
|
166
|
+
for category, pattern, replacement in self._steps:
|
|
167
|
+
result, count = pattern.subn(replacement, result)
|
|
168
|
+
self._counts[category] += count
|
|
169
|
+
return result
|
|
170
|
+
|
|
171
|
+
def redact(self, obj: T) -> T:
|
|
172
|
+
"""Return a redacted copy of ``obj``.
|
|
173
|
+
|
|
174
|
+
Strings are redacted; dataclasses (frozen ones too), tuples, lists and dicts are
|
|
175
|
+
copied with their contents redacted; enums, numbers, booleans and ``None`` are
|
|
176
|
+
returned unchanged. The original object is never modified.
|
|
177
|
+
"""
|
|
178
|
+
return cast(T, self._redact(obj))
|
|
179
|
+
|
|
180
|
+
@property
|
|
181
|
+
def summary(self) -> dict[str, int]:
|
|
182
|
+
"""How many values of each kind this redactor has removed so far."""
|
|
183
|
+
return {category: count for category, count in self._counts.items() if count}
|
|
184
|
+
|
|
185
|
+
def _redact(self, obj: Any) -> Any:
|
|
186
|
+
if isinstance(obj, Enum): # StrEnum members are str; keep them as they are
|
|
187
|
+
return obj
|
|
188
|
+
if isinstance(obj, str):
|
|
189
|
+
return self.text(obj)
|
|
190
|
+
if is_dataclass(obj) and not isinstance(obj, type):
|
|
191
|
+
changes = {f.name: self._redact(getattr(obj, f.name)) for f in fields(obj) if f.init}
|
|
192
|
+
return replace(obj, **changes)
|
|
193
|
+
if isinstance(obj, tuple):
|
|
194
|
+
return tuple(self._redact(item) for item in obj)
|
|
195
|
+
if isinstance(obj, list):
|
|
196
|
+
return [self._redact(item) for item in obj]
|
|
197
|
+
if isinstance(obj, dict):
|
|
198
|
+
return {self._redact(key): self._redact(item) for key, item in obj.items()}
|
|
199
|
+
return obj
|
|
200
|
+
|
|
201
|
+
def _url_credentials(self, match: re.Match[str]) -> str:
|
|
202
|
+
scheme, credentials = match.group(1), match.group(2)
|
|
203
|
+
if credentials == _REDACTED:
|
|
204
|
+
return match.group(0)
|
|
205
|
+
self._counts["url_credentials"] += 1
|
|
206
|
+
return f"{scheme}{_REDACTED}@"
|
|
207
|
+
|
|
208
|
+
def _secret_assignments(self, text: str) -> str:
|
|
209
|
+
# Scanned by hand rather than with re.sub: the value of a harmless name must not be
|
|
210
|
+
# consumed, or "env: DB_PASSWORD=hunter2" would hide the password inside the value
|
|
211
|
+
# of "env". After a harmless name the scan resumes right after its separator.
|
|
212
|
+
parts: list[str] = []
|
|
213
|
+
pos = 0
|
|
214
|
+
while match := _NAME_AND_SEPARATOR.search(text, pos):
|
|
215
|
+
parts.append(text[pos : match.end()])
|
|
216
|
+
pos = match.end()
|
|
217
|
+
value = _VALUE.match(text, pos)
|
|
218
|
+
if (
|
|
219
|
+
value is not None
|
|
220
|
+
and _is_secret_name(match.group(1))
|
|
221
|
+
and not value.group(0).startswith("<redacted")
|
|
222
|
+
):
|
|
223
|
+
parts.append(_REDACTED)
|
|
224
|
+
pos = value.end()
|
|
225
|
+
self._counts["secret_value"] += 1
|
|
226
|
+
parts.append(text[pos:])
|
|
227
|
+
return "".join(parts)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def redact_snapshot(snapshot: Snapshot, context: RedactionContext | None = None) -> Snapshot:
|
|
231
|
+
"""The snapshot redacted for output, with a record of what was removed."""
|
|
232
|
+
from gpu_stack_inspector.models import RedactionCount
|
|
233
|
+
|
|
234
|
+
redactor = Redactor(context or RedactionContext.current())
|
|
235
|
+
redacted = redactor.redact(snapshot)
|
|
236
|
+
counts = tuple(RedactionCount(kind, n) for kind, n in sorted(redactor.summary.items()))
|
|
237
|
+
return replace(redacted, redactions=counts)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Programs that run inside a target interpreter, and the code that runs them."""
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"""Running the metadata probe inside a target interpreter and reading its answer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import functools
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
import tempfile
|
|
9
|
+
from importlib.resources import files
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from gpu_stack_inspector.collectors.python_env import PROBE, interpreter_path
|
|
14
|
+
from gpu_stack_inspector.models import (
|
|
15
|
+
Distribution,
|
|
16
|
+
FailureKind,
|
|
17
|
+
InterpreterFacts,
|
|
18
|
+
ProbeFailure,
|
|
19
|
+
TorchFacts,
|
|
20
|
+
)
|
|
21
|
+
from gpu_stack_inspector.runner import CommandResult, CommandRunner, Outcome, probe_failure
|
|
22
|
+
|
|
23
|
+
PROTOCOL = 1
|
|
24
|
+
BEGIN, END = "GSI-PROBE-BEGIN", "GSI-PROBE-END"
|
|
25
|
+
TIMEOUT = 8.0
|
|
26
|
+
"""Starting an interpreter and reading its metadata takes about a second, even in large
|
|
27
|
+
environments; a hung startup (a broken .pth file, a network mount) must not hang us."""
|
|
28
|
+
|
|
29
|
+
_EXCEPTION_LINE = re.compile(r"[A-Za-z_][\w.]*(?:Error|Exception|Exit|Interrupt)\b.*")
|
|
30
|
+
_DETAIL_LIMIT = 300
|
|
31
|
+
|
|
32
|
+
_STRING_FIELDS = ("version", "implementation", "executable", "prefix", "base_prefix", "platform")
|
|
33
|
+
_BOOL_FIELDS = ("conda", "user_site", "pythonpath_set")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@functools.cache
|
|
37
|
+
def program_source() -> str:
|
|
38
|
+
"""The probe program, passed to the target as ``python -c``."""
|
|
39
|
+
resource = files("gpu_stack_inspector.probes").joinpath("metadata_program.py")
|
|
40
|
+
return resource.read_text(encoding="utf-8")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def probe_interpreter(
|
|
44
|
+
runner: CommandRunner,
|
|
45
|
+
label: str,
|
|
46
|
+
raw_path: str,
|
|
47
|
+
*,
|
|
48
|
+
cwd: Path | None = None,
|
|
49
|
+
timeout: float = TIMEOUT,
|
|
50
|
+
) -> InterpreterFacts:
|
|
51
|
+
"""Run the metadata probe in the interpreter named by ``raw_path``.
|
|
52
|
+
|
|
53
|
+
The target's environment is passed on unchanged: its variables are part of what is
|
|
54
|
+
being diagnosed. Only the working directory is replaced, see below.
|
|
55
|
+
"""
|
|
56
|
+
path, failure = interpreter_path(raw_path, cwd=cwd)
|
|
57
|
+
if path is None:
|
|
58
|
+
return InterpreterFacts(label, raw_path, failures=(failure,) if failure else ())
|
|
59
|
+
argv = [str(path), "-c", program_source()]
|
|
60
|
+
# With -c, Python puts the working directory first on sys.path. Running in a new
|
|
61
|
+
# empty directory means a module planted where the user runs the tool (a json.py,
|
|
62
|
+
# say) is never imported by the probe.
|
|
63
|
+
with tempfile.TemporaryDirectory(prefix="gsi-probe-") as empty:
|
|
64
|
+
result = runner.run(argv, timeout=timeout, cwd=Path(empty))
|
|
65
|
+
if result.outcome is Outcome.NONZERO:
|
|
66
|
+
return InterpreterFacts(label, str(path), failures=(_crash(path, result),))
|
|
67
|
+
failure = probe_failure(PROBE, result)
|
|
68
|
+
if failure is not None:
|
|
69
|
+
return InterpreterFacts(label, str(path), failures=(failure,))
|
|
70
|
+
return parse_probe_output(label, str(path), result.stdout, truncated=result.stdout_truncated)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def parse_probe_output(
|
|
74
|
+
label: str, path: str, stdout: str, *, truncated: bool = False
|
|
75
|
+
) -> InterpreterFacts:
|
|
76
|
+
"""Read the JSON document between the markers; anything unexpected becomes unknown."""
|
|
77
|
+
start = stdout.find(BEGIN)
|
|
78
|
+
end = stdout.find(END, start + len(BEGIN)) if start != -1 else -1
|
|
79
|
+
if start == -1 or end == -1:
|
|
80
|
+
if truncated:
|
|
81
|
+
return _failed(
|
|
82
|
+
label, path, "the probe's output exceeded the output limit and was cut off"
|
|
83
|
+
)
|
|
84
|
+
return _failed(label, path, "the interpreter printed no probe result")
|
|
85
|
+
try:
|
|
86
|
+
document = json.loads(stdout[start + len(BEGIN) : end])
|
|
87
|
+
except ValueError:
|
|
88
|
+
return _failed(label, path, "the probe result is not valid JSON")
|
|
89
|
+
if not isinstance(document, dict):
|
|
90
|
+
return _failed(label, path, "the probe result is not a JSON object")
|
|
91
|
+
if document.get("protocol") != PROTOCOL:
|
|
92
|
+
return _failed(label, path, f"unexpected probe protocol {document.get('protocol')!r}")
|
|
93
|
+
|
|
94
|
+
unexpected: list[str] = []
|
|
95
|
+
values = _python_values(document.get("python"), unexpected)
|
|
96
|
+
distributions = _distributions(document.get("distributions"), unexpected)
|
|
97
|
+
torch = _torch(document.get("torch"), unexpected)
|
|
98
|
+
failures: list[ProbeFailure] = []
|
|
99
|
+
if unexpected:
|
|
100
|
+
detail = "unexpected values for: " + ", ".join(dict.fromkeys(unexpected))
|
|
101
|
+
failures.append(_malformed(detail))
|
|
102
|
+
errors = document.get("errors")
|
|
103
|
+
if isinstance(errors, list):
|
|
104
|
+
failures.extend(
|
|
105
|
+
ProbeFailure(PROBE, FailureKind.PROBE_ERROR, error)
|
|
106
|
+
for error in errors
|
|
107
|
+
if isinstance(error, str)
|
|
108
|
+
)
|
|
109
|
+
return InterpreterFacts(
|
|
110
|
+
label,
|
|
111
|
+
path,
|
|
112
|
+
**values,
|
|
113
|
+
distributions=tuple(distributions),
|
|
114
|
+
torch=torch,
|
|
115
|
+
failures=tuple(failures),
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _python_values(section: object, unexpected: list[str]) -> dict[str, Any]:
|
|
120
|
+
values: dict[str, Any] = dict.fromkeys((*_STRING_FIELDS, *_BOOL_FIELDS))
|
|
121
|
+
if section is None:
|
|
122
|
+
return values
|
|
123
|
+
if not isinstance(section, dict):
|
|
124
|
+
unexpected.append("python")
|
|
125
|
+
return values
|
|
126
|
+
for keys, kind in ((_STRING_FIELDS, str), (_BOOL_FIELDS, bool)):
|
|
127
|
+
for key in keys:
|
|
128
|
+
value = section.get(key)
|
|
129
|
+
if value is None or isinstance(value, kind):
|
|
130
|
+
values[key] = value
|
|
131
|
+
else:
|
|
132
|
+
unexpected.append(key)
|
|
133
|
+
return values
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _distributions(section: object, unexpected: list[str]) -> list[Distribution]:
|
|
137
|
+
if section is None:
|
|
138
|
+
return []
|
|
139
|
+
if not isinstance(section, list):
|
|
140
|
+
unexpected.append("distributions")
|
|
141
|
+
return []
|
|
142
|
+
found = []
|
|
143
|
+
for entry in section:
|
|
144
|
+
if (
|
|
145
|
+
isinstance(entry, dict)
|
|
146
|
+
and isinstance(entry.get("name"), str)
|
|
147
|
+
and isinstance(entry.get("version"), str)
|
|
148
|
+
and isinstance(entry.get("location"), str | None)
|
|
149
|
+
):
|
|
150
|
+
found.append(Distribution(entry["name"], entry["version"], entry.get("location")))
|
|
151
|
+
else:
|
|
152
|
+
unexpected.append("distributions")
|
|
153
|
+
return found
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def python_error_line(stderr: str) -> str:
|
|
157
|
+
"""The line of a Python crash that names the problem, or "" if there is none.
|
|
158
|
+
|
|
159
|
+
The exception at the end of a traceback, else the "Fatal Python error" line, else the
|
|
160
|
+
first line of stderr.
|
|
161
|
+
"""
|
|
162
|
+
lines = [line.strip() for line in stderr.splitlines() if line.strip()]
|
|
163
|
+
reason = next((line for line in reversed(lines) if _EXCEPTION_LINE.fullmatch(line)), None)
|
|
164
|
+
if reason is None:
|
|
165
|
+
reason = next((line for line in lines if line.startswith("Fatal Python error")), None)
|
|
166
|
+
if reason is None:
|
|
167
|
+
reason = lines[0] if lines else ""
|
|
168
|
+
return reason
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _crash(path: Path, result: CommandResult) -> ProbeFailure:
|
|
172
|
+
"""Explain a crashed interpreter by the line that names the problem.
|
|
173
|
+
|
|
174
|
+
A traceback starts with "Traceback (most recent call last):", which says nothing;
|
|
175
|
+
the exception is on its last line. "Fatal Python error" output (a broken standard
|
|
176
|
+
library, for instance) names the underlying exception further down.
|
|
177
|
+
"""
|
|
178
|
+
reason = python_error_line(result.stderr)
|
|
179
|
+
detail = f"{path} exited with code {result.returncode}"
|
|
180
|
+
if reason:
|
|
181
|
+
detail += f": {reason}"
|
|
182
|
+
if reason.startswith("SyntaxError"):
|
|
183
|
+
detail += " (the probe needs Python 3.8 or newer)"
|
|
184
|
+
if len(detail) > _DETAIL_LIMIT:
|
|
185
|
+
detail = detail[: _DETAIL_LIMIT - 1] + "…"
|
|
186
|
+
return ProbeFailure(PROBE, FailureKind.NONZERO_EXIT, detail)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _torch(section: object, unexpected: list[str]) -> TorchFacts | None:
|
|
190
|
+
if section is None:
|
|
191
|
+
return None
|
|
192
|
+
if not isinstance(section, dict) or not isinstance(section.get("version"), str):
|
|
193
|
+
unexpected.append("torch")
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
def optional(value: object, kind: type, key: str) -> Any:
|
|
197
|
+
if value is None or isinstance(value, kind):
|
|
198
|
+
return value
|
|
199
|
+
unexpected.append(f"torch.{key}")
|
|
200
|
+
return None
|
|
201
|
+
|
|
202
|
+
def strings(value: object, key: str) -> tuple[str, ...] | None:
|
|
203
|
+
if value is None:
|
|
204
|
+
return None
|
|
205
|
+
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
|
206
|
+
return tuple(value)
|
|
207
|
+
unexpected.append(f"torch.{key}")
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
libraries = optional(section.get("libraries"), dict, "libraries") or {}
|
|
211
|
+
version_py = optional(section.get("version_py"), dict, "version_py")
|
|
212
|
+
return TorchFacts(
|
|
213
|
+
version=section["version"],
|
|
214
|
+
location=optional(section.get("location"), str, "location"),
|
|
215
|
+
requires_nvidia=strings(section.get("requires_nvidia"), "requires_nvidia") or (),
|
|
216
|
+
cuda_library=optional(libraries.get("cuda"), bool, "libraries.cuda"),
|
|
217
|
+
hip_library=optional(libraries.get("hip"), bool, "libraries.hip"),
|
|
218
|
+
version_py_found=version_py is not None,
|
|
219
|
+
built_cuda=optional((version_py or {}).get("cuda"), str, "version_py.cuda"),
|
|
220
|
+
built_hip=optional((version_py or {}).get("hip"), str, "version_py.hip"),
|
|
221
|
+
conda_records=strings(section.get("conda_records"), "conda_records"),
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _failed(label: str, path: str, detail: str) -> InterpreterFacts:
|
|
226
|
+
return InterpreterFacts(label, path, failures=(_malformed(detail),))
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _malformed(detail: str) -> ProbeFailure:
|
|
230
|
+
return ProbeFailure(PROBE, FailureKind.MALFORMED_OUTPUT, detail)
|