kernelvet 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kernelvet/__init__.py +10 -0
- kernelvet/__main__.py +3 -0
- kernelvet/benchmark/__init__.py +1 -0
- kernelvet/benchmark/engine.py +180 -0
- kernelvet/benchmark/workloads.py +153 -0
- kernelvet/cli/__init__.py +1 -0
- kernelvet/cli/app.py +64 -0
- kernelvet/cli/commands/__init__.py +1 -0
- kernelvet/cli/commands/diagnose.py +244 -0
- kernelvet/cli/commands/migrate.py +334 -0
- kernelvet/cli/commands/misc.py +216 -0
- kernelvet/cli/output.py +110 -0
- kernelvet/compatibility/__init__.py +1 -0
- kernelvet/compatibility/data/rocm_matrix.json +123 -0
- kernelvet/compatibility/engine.py +24 -0
- kernelvet/compatibility/matrix.py +128 -0
- kernelvet/compatibility/validators.py +405 -0
- kernelvet/config/__init__.py +10 -0
- kernelvet/config/settings.py +112 -0
- kernelvet/diagnose/__init__.py +1 -0
- kernelvet/diagnose/engine.py +296 -0
- kernelvet/diagnose/hardware.py +336 -0
- kernelvet/diagnose/runtime.py +332 -0
- kernelvet/diagnose/software.py +221 -0
- kernelvet/diagnose/system.py +67 -0
- kernelvet/execution/__init__.py +1 -0
- kernelvet/execution/compiler.py +275 -0
- kernelvet/execution/runner.py +125 -0
- kernelvet/execution/sandbox.py +51 -0
- kernelvet/history/__init__.py +3 -0
- kernelvet/history/store.py +157 -0
- kernelvet/main.py +19 -0
- kernelvet/migration/__init__.py +17 -0
- kernelvet/migration/analyzer.py +78 -0
- kernelvet/migration/auditor.py +89 -0
- kernelvet/migration/hipify.py +268 -0
- kernelvet/migration/planner.py +128 -0
- kernelvet/migration/rules.py +564 -0
- kernelvet/migration/scanner.py +234 -0
- kernelvet/migration/workflow.py +135 -0
- kernelvet/models/__init__.py +35 -0
- kernelvet/models/common.py +102 -0
- kernelvet/models/migration.py +236 -0
- kernelvet/models/report.py +126 -0
- kernelvet/models/system.py +89 -0
- kernelvet/reporting/__init__.py +80 -0
- kernelvet/reporting/common.py +74 -0
- kernelvet/reporting/html.py +149 -0
- kernelvet/reporting/json.py +20 -0
- kernelvet/reporting/loader.py +48 -0
- kernelvet/reporting/markdown.py +256 -0
- kernelvet/reporting/sarif.py +79 -0
- kernelvet/reporting/terminal.py +276 -0
- kernelvet/tui/__init__.py +1 -0
- kernelvet/tui/app.py +177 -0
- kernelvet/tui/screens/__init__.py +1 -0
- kernelvet/tui/screens/base.py +34 -0
- kernelvet/tui/screens/devices.py +211 -0
- kernelvet/tui/screens/diagnose.py +212 -0
- kernelvet/tui/screens/help.py +47 -0
- kernelvet/tui/screens/migration.py +573 -0
- kernelvet/tui/screens/overview.py +195 -0
- kernelvet/tui/screens/tools.py +357 -0
- kernelvet/tui/state.py +55 -0
- kernelvet/tui/widgets/__init__.py +1 -0
- kernelvet/tui/widgets/common.py +82 -0
- kernelvet/tui/workers/__init__.py +25 -0
- kernelvet/utils/__init__.py +1 -0
- kernelvet/utils/paths.py +48 -0
- kernelvet/utils/proc.py +178 -0
- kernelvet/utils/redact.py +47 -0
- kernelvet/utils/versions.py +34 -0
- kernelvet-0.1.0.dist-info/METADATA +275 -0
- kernelvet-0.1.0.dist-info/RECORD +77 -0
- kernelvet-0.1.0.dist-info/WHEEL +4 -0
- kernelvet-0.1.0.dist-info/entry_points.txt +2 -0
- kernelvet-0.1.0.dist-info/licenses/LICENSE +197 -0
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""Runtime verification: actually execute code on the GPU.
|
|
2
|
+
|
|
3
|
+
Two probes:
|
|
4
|
+
|
|
5
|
+
* a HIP C++ program compiled with ``hipcc`` that initialises the runtime,
|
|
6
|
+
enumerates devices, allocates memory, copies data both ways, launches a
|
|
7
|
+
vector-add kernel and verifies the result on the host;
|
|
8
|
+
* a PyTorch script that runs tensor operations on the GPU and compares them
|
|
9
|
+
with a CPU reference.
|
|
10
|
+
|
|
11
|
+
Each probe step becomes its own check with ``Evidence.EXECUTED``. When a probe
|
|
12
|
+
cannot run, every step is SKIPPED with the reason. Nothing is assumed to pass.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
import time
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
from kernelvet.models.common import CheckResult, CheckStatus, Evidence
|
|
24
|
+
from kernelvet.models.system import TorchInfo
|
|
25
|
+
from kernelvet.utils.proc import HostContext
|
|
26
|
+
|
|
27
|
+
HIP_PROBE_SOURCE = r"""
|
|
28
|
+
#include <hip/hip_runtime.h>
|
|
29
|
+
#include <cstdio>
|
|
30
|
+
#include <cstdlib>
|
|
31
|
+
#include <cmath>
|
|
32
|
+
#include <vector>
|
|
33
|
+
|
|
34
|
+
#define REPORT(step, ok, ...) do { std::printf("STEP %s %s ", step, (ok) ? "OK" : "FAIL"); \
|
|
35
|
+
std::printf(__VA_ARGS__); std::printf("\n"); std::fflush(stdout); } while (0)
|
|
36
|
+
|
|
37
|
+
__global__ void vector_add(const float* a, const float* b, float* c, int n) {
|
|
38
|
+
int i = blockIdx.x * blockDim.x + threadIdx.x;
|
|
39
|
+
if (i < n) c[i] = a[i] + b[i];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
int main() {
|
|
43
|
+
int count = 0;
|
|
44
|
+
hipError_t err = hipGetDeviceCount(&count);
|
|
45
|
+
REPORT("init", err == hipSuccess, "%s", hipGetErrorString(err));
|
|
46
|
+
if (err != hipSuccess) return 2;
|
|
47
|
+
REPORT("enumerate", count > 0, "devices=%d", count);
|
|
48
|
+
if (count == 0) return 3;
|
|
49
|
+
for (int d = 0; d < count; ++d) {
|
|
50
|
+
hipDeviceProp_t p;
|
|
51
|
+
if (hipGetDeviceProperties(&p, d) == hipSuccess)
|
|
52
|
+
std::printf("DEVICE %d %s|%s|%zu\n", d, p.gcnArchName, p.name, (size_t)p.totalGlobalMem);
|
|
53
|
+
}
|
|
54
|
+
err = hipSetDevice(0);
|
|
55
|
+
if (err != hipSuccess) { REPORT("alloc", false, "hipSetDevice: %s", hipGetErrorString(err)); return 4; }
|
|
56
|
+
|
|
57
|
+
const int n = 1 << 20;
|
|
58
|
+
const size_t bytes = n * sizeof(float);
|
|
59
|
+
std::vector<float> a(n), b(n), c(n, 0.f);
|
|
60
|
+
for (int i = 0; i < n; ++i) { a[i] = (float)i * 0.5f; b[i] = (float)(n - i); }
|
|
61
|
+
float *da = nullptr, *db = nullptr, *dc = nullptr;
|
|
62
|
+
err = hipMalloc(&da, bytes);
|
|
63
|
+
if (err == hipSuccess) err = hipMalloc(&db, bytes);
|
|
64
|
+
if (err == hipSuccess) err = hipMalloc(&dc, bytes);
|
|
65
|
+
REPORT("alloc", err == hipSuccess, "%zu bytes x3: %s", bytes, hipGetErrorString(err));
|
|
66
|
+
if (err != hipSuccess) return 5;
|
|
67
|
+
|
|
68
|
+
err = hipMemcpy(da, a.data(), bytes, hipMemcpyHostToDevice);
|
|
69
|
+
if (err == hipSuccess) err = hipMemcpy(db, b.data(), bytes, hipMemcpyHostToDevice);
|
|
70
|
+
REPORT("h2d", err == hipSuccess, "%s", hipGetErrorString(err));
|
|
71
|
+
if (err != hipSuccess) return 6;
|
|
72
|
+
|
|
73
|
+
vector_add<<<(n + 255) / 256, 256>>>(da, db, dc, n);
|
|
74
|
+
err = hipGetLastError();
|
|
75
|
+
if (err == hipSuccess) err = hipDeviceSynchronize();
|
|
76
|
+
REPORT("kernel", err == hipSuccess, "%s", hipGetErrorString(err));
|
|
77
|
+
if (err != hipSuccess) return 7;
|
|
78
|
+
|
|
79
|
+
err = hipMemcpy(c.data(), dc, bytes, hipMemcpyDeviceToHost);
|
|
80
|
+
REPORT("d2h", err == hipSuccess, "%s", hipGetErrorString(err));
|
|
81
|
+
if (err != hipSuccess) return 8;
|
|
82
|
+
|
|
83
|
+
int bad = 0;
|
|
84
|
+
for (int i = 0; i < n; ++i) if (std::fabs(c[i] - (a[i] + b[i])) > 1e-3f) ++bad;
|
|
85
|
+
REPORT("verify", bad == 0, "mismatches=%d of %d", bad, n);
|
|
86
|
+
hipFree(da); hipFree(db); hipFree(dc);
|
|
87
|
+
return bad == 0 ? 0 : 9;
|
|
88
|
+
}
|
|
89
|
+
"""
|
|
90
|
+
|
|
91
|
+
HIP_STEPS: list[tuple[str, str, str]] = [
|
|
92
|
+
("init", "runtime.hip_init", "HIP Initialization"),
|
|
93
|
+
("enumerate", "runtime.device_enum", "Device Enumeration"),
|
|
94
|
+
("alloc", "runtime.memory_alloc", "GPU Memory Allocation"),
|
|
95
|
+
("h2d", "runtime.memcpy_h2d", "Host-to-Device Transfer"),
|
|
96
|
+
("kernel", "runtime.kernel_exec", "Kernel Execution"),
|
|
97
|
+
("d2h", "runtime.memcpy_d2h", "Device-to-Host Transfer"),
|
|
98
|
+
("verify", "runtime.vector_add_verify", "Vector Add Result Verification"),
|
|
99
|
+
]
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _skip_all(reason: str, remediation: str | None = None) -> list[CheckResult]:
|
|
103
|
+
return [
|
|
104
|
+
CheckResult(
|
|
105
|
+
id=cid,
|
|
106
|
+
name=name,
|
|
107
|
+
category="runtime",
|
|
108
|
+
status=CheckStatus.SKIPPED,
|
|
109
|
+
evidence=Evidence.NONE,
|
|
110
|
+
summary=reason,
|
|
111
|
+
remediation=remediation,
|
|
112
|
+
)
|
|
113
|
+
for _, cid, name in HIP_STEPS
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def parse_probe_output(stdout: str) -> tuple[dict[str, tuple[bool, str]], list[str]]:
|
|
118
|
+
steps: dict[str, tuple[bool, str]] = {}
|
|
119
|
+
devices: list[str] = []
|
|
120
|
+
for line in stdout.splitlines():
|
|
121
|
+
parts = line.split(" ", 3)
|
|
122
|
+
if len(parts) >= 3 and parts[0] == "STEP":
|
|
123
|
+
steps[parts[1]] = (parts[2] == "OK", parts[3] if len(parts) > 3 else "")
|
|
124
|
+
elif line.startswith("DEVICE "):
|
|
125
|
+
devices.append(line[len("DEVICE ") :])
|
|
126
|
+
return steps, devices
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def offload_arch_flags(targets: list[str]) -> list[str]:
|
|
130
|
+
uniq = sorted({t.split(":")[0] for t in targets if t and t.startswith("gfx")})
|
|
131
|
+
return [f"--offload-arch={t}" for t in uniq]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def run_hip_probe(
|
|
135
|
+
ctx: HostContext,
|
|
136
|
+
*,
|
|
137
|
+
gfx_targets: list[str] | None = None,
|
|
138
|
+
workdir: Path | None = None,
|
|
139
|
+
compile_timeout: float = 180.0,
|
|
140
|
+
run_timeout: float = 60.0,
|
|
141
|
+
) -> list[CheckResult]:
|
|
142
|
+
if not ctx.which("hipcc"):
|
|
143
|
+
return _skip_all(
|
|
144
|
+
"hipcc not found; the HIP probe needs a ROCm toolchain",
|
|
145
|
+
"Install ROCm (HIP SDK) and ensure hipcc is on PATH",
|
|
146
|
+
)
|
|
147
|
+
with tempfile.TemporaryDirectory(prefix="kernelvet-probe-", dir=workdir) as tmp:
|
|
148
|
+
src = Path(tmp) / "probe.hip"
|
|
149
|
+
exe = Path(tmp) / "probe"
|
|
150
|
+
src.write_text(HIP_PROBE_SOURCE)
|
|
151
|
+
argv = ["hipcc", "-O2", *offload_arch_flags(gfx_targets or []), "-o", str(exe), str(src)]
|
|
152
|
+
t0 = time.monotonic()
|
|
153
|
+
build = ctx.run(argv, timeout=compile_timeout)
|
|
154
|
+
if not build.ok:
|
|
155
|
+
return _skip_all(
|
|
156
|
+
f"HIP probe did not compile: {build.describe_failure()}",
|
|
157
|
+
"Check that hipcc works: `hipcc --version`; see the compiler output in the JSON report",
|
|
158
|
+
)
|
|
159
|
+
compile_ms = (time.monotonic() - t0) * 1000
|
|
160
|
+
t1 = time.monotonic()
|
|
161
|
+
res = ctx.run([str(exe)], timeout=run_timeout)
|
|
162
|
+
run_ms = (time.monotonic() - t1) * 1000
|
|
163
|
+
|
|
164
|
+
steps, devices = parse_probe_output(res.stdout)
|
|
165
|
+
checks: list[CheckResult] = []
|
|
166
|
+
blocked_by: str | None = None
|
|
167
|
+
for key, cid, name in HIP_STEPS:
|
|
168
|
+
if key in steps:
|
|
169
|
+
ok, detail = steps[key]
|
|
170
|
+
checks.append(
|
|
171
|
+
CheckResult(
|
|
172
|
+
id=cid,
|
|
173
|
+
name=name,
|
|
174
|
+
category="runtime",
|
|
175
|
+
status=CheckStatus.PASS if ok else CheckStatus.FAIL,
|
|
176
|
+
evidence=Evidence.EXECUTED,
|
|
177
|
+
summary=detail.strip() or ("ok" if ok else "failed"),
|
|
178
|
+
detail=(
|
|
179
|
+
f"compiled in {compile_ms:.0f} ms, ran in {run_ms:.0f} ms"
|
|
180
|
+
+ (f"; devices: {'; '.join(devices)}" if devices and key == "enumerate" else "")
|
|
181
|
+
),
|
|
182
|
+
remediation=None if ok else _runtime_remediation(key),
|
|
183
|
+
duration_ms=run_ms,
|
|
184
|
+
)
|
|
185
|
+
)
|
|
186
|
+
if not ok:
|
|
187
|
+
blocked_by = name
|
|
188
|
+
else:
|
|
189
|
+
reason = (
|
|
190
|
+
f"not reached: {blocked_by} failed"
|
|
191
|
+
if blocked_by
|
|
192
|
+
else f"probe ended early ({res.describe_failure()})"
|
|
193
|
+
)
|
|
194
|
+
checks.append(
|
|
195
|
+
CheckResult(
|
|
196
|
+
id=cid,
|
|
197
|
+
name=name,
|
|
198
|
+
category="runtime",
|
|
199
|
+
status=CheckStatus.SKIPPED if blocked_by else CheckStatus.FAIL,
|
|
200
|
+
evidence=Evidence.EXECUTED if not blocked_by else Evidence.NONE,
|
|
201
|
+
summary=reason,
|
|
202
|
+
remediation=None if blocked_by else _runtime_remediation(key),
|
|
203
|
+
)
|
|
204
|
+
)
|
|
205
|
+
blocked_by = blocked_by or name
|
|
206
|
+
return checks
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _runtime_remediation(step: str) -> str:
|
|
210
|
+
return {
|
|
211
|
+
"init": "HIP could not initialise. Check `rocminfo` works, that /dev/kfd exists, and that "
|
|
212
|
+
"your user is in the `render` and `video` groups",
|
|
213
|
+
"enumerate": "No HIP devices visible. Check HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES and "
|
|
214
|
+
"that the amdgpu driver is loaded",
|
|
215
|
+
"alloc": "Device memory allocation failed; check free VRAM with `amd-smi metric --mem-usage`",
|
|
216
|
+
"kernel": "The kernel failed to launch or run. The binary may lack code for this GPU "
|
|
217
|
+
"architecture; rebuild with --offload-arch=<your gfx target>",
|
|
218
|
+
}.get(step, "See the probe output in the JSON report")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
TORCH_RUNTIME_PROBE = r"""
|
|
222
|
+
import json, sys, time
|
|
223
|
+
out = {"steps": {}}
|
|
224
|
+
def step(name, ok, detail=""):
|
|
225
|
+
out["steps"][name] = {"ok": bool(ok), "detail": str(detail)}
|
|
226
|
+
try:
|
|
227
|
+
import torch
|
|
228
|
+
avail = torch.cuda.is_available()
|
|
229
|
+
step("available", avail, f"device_count={torch.cuda.device_count()}")
|
|
230
|
+
if not avail:
|
|
231
|
+
print(json.dumps(out)); sys.exit(0)
|
|
232
|
+
names = [torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count())]
|
|
233
|
+
out["devices"] = names
|
|
234
|
+
torch.manual_seed(0)
|
|
235
|
+
a = torch.randn(512, 512)
|
|
236
|
+
b = torch.randn(512, 512)
|
|
237
|
+
t0 = time.perf_counter()
|
|
238
|
+
ga, gb = a.to("cuda"), b.to("cuda")
|
|
239
|
+
gc = ga @ gb
|
|
240
|
+
torch.cuda.synchronize()
|
|
241
|
+
dt = (time.perf_counter() - t0) * 1000
|
|
242
|
+
ref = a @ b
|
|
243
|
+
diff = (gc.cpu() - ref).abs().max().item()
|
|
244
|
+
step("tensor_ops", diff < 1e-2, f"512x512 matmul on {names[0]}, max |diff| vs CPU = {diff:.2e}, {dt:.1f} ms")
|
|
245
|
+
except Exception as exc:
|
|
246
|
+
step("error", False, f"{type(exc).__name__}: {exc}")
|
|
247
|
+
print(json.dumps(out))
|
|
248
|
+
"""
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def run_torch_probe(
|
|
252
|
+
ctx: HostContext, torch_info: TorchInfo | None, python: str | None = None
|
|
253
|
+
) -> list[CheckResult]:
|
|
254
|
+
def mk(
|
|
255
|
+
cid: str,
|
|
256
|
+
name: str,
|
|
257
|
+
status: CheckStatus,
|
|
258
|
+
summary: str,
|
|
259
|
+
ev: Evidence = Evidence.NONE,
|
|
260
|
+
remediation: str | None = None,
|
|
261
|
+
) -> CheckResult:
|
|
262
|
+
return CheckResult(
|
|
263
|
+
id=cid,
|
|
264
|
+
name=name,
|
|
265
|
+
category="runtime",
|
|
266
|
+
status=status,
|
|
267
|
+
evidence=ev,
|
|
268
|
+
summary=summary,
|
|
269
|
+
remediation=remediation,
|
|
270
|
+
)
|
|
271
|
+
|
|
272
|
+
avail_id, avail_name = "runtime.torch_gpu", "PyTorch GPU Visibility"
|
|
273
|
+
ops_id, ops_name = "runtime.torch_tensor_ops", "PyTorch GPU Tensor Operations"
|
|
274
|
+
if torch_info is None or not torch_info.installed:
|
|
275
|
+
reason = "PyTorch is not installed in the checked interpreter"
|
|
276
|
+
return [
|
|
277
|
+
mk(avail_id, avail_name, CheckStatus.SKIPPED, reason),
|
|
278
|
+
mk(ops_id, ops_name, CheckStatus.SKIPPED, reason),
|
|
279
|
+
]
|
|
280
|
+
if not torch_info.is_rocm_build:
|
|
281
|
+
build = f"CUDA {torch_info.cuda_version}" if torch_info.cuda_version else "CPU-only"
|
|
282
|
+
reason = f"PyTorch {torch_info.version} is a {build} build, not a ROCm build"
|
|
283
|
+
fix = "Install a ROCm build of PyTorch, e.g. from https://pytorch.org/get-started/locally/"
|
|
284
|
+
return [
|
|
285
|
+
mk(avail_id, avail_name, CheckStatus.SKIPPED, reason, remediation=fix),
|
|
286
|
+
mk(ops_id, ops_name, CheckStatus.SKIPPED, reason),
|
|
287
|
+
]
|
|
288
|
+
res = ctx.run([python or sys.executable, "-c", TORCH_RUNTIME_PROBE], timeout=180)
|
|
289
|
+
try:
|
|
290
|
+
data = json.loads(res.stdout.strip().splitlines()[-1])
|
|
291
|
+
except (json.JSONDecodeError, IndexError):
|
|
292
|
+
msg = res.describe_failure()
|
|
293
|
+
return [
|
|
294
|
+
mk(avail_id, avail_name, CheckStatus.FAIL, msg, Evidence.EXECUTED),
|
|
295
|
+
mk(ops_id, ops_name, CheckStatus.SKIPPED, "not reached"),
|
|
296
|
+
]
|
|
297
|
+
steps = data.get("steps", {})
|
|
298
|
+
checks = []
|
|
299
|
+
a = steps.get("available")
|
|
300
|
+
if a is None:
|
|
301
|
+
err = steps.get("error", {}).get("detail", "unknown error")
|
|
302
|
+
checks.append(mk(avail_id, avail_name, CheckStatus.FAIL, err, Evidence.EXECUTED))
|
|
303
|
+
else:
|
|
304
|
+
checks.append(
|
|
305
|
+
mk(
|
|
306
|
+
avail_id,
|
|
307
|
+
avail_name,
|
|
308
|
+
CheckStatus.PASS if a["ok"] else CheckStatus.FAIL,
|
|
309
|
+
a["detail"],
|
|
310
|
+
Evidence.EXECUTED,
|
|
311
|
+
None
|
|
312
|
+
if a["ok"]
|
|
313
|
+
else "torch.cuda.is_available() is False; check HIP_VISIBLE_DEVICES "
|
|
314
|
+
"and that the PyTorch ROCm version matches the installed ROCm",
|
|
315
|
+
)
|
|
316
|
+
)
|
|
317
|
+
t = steps.get("tensor_ops")
|
|
318
|
+
if t is not None:
|
|
319
|
+
checks.append(
|
|
320
|
+
mk(
|
|
321
|
+
ops_id,
|
|
322
|
+
ops_name,
|
|
323
|
+
CheckStatus.PASS if t["ok"] else CheckStatus.FAIL,
|
|
324
|
+
t["detail"],
|
|
325
|
+
Evidence.EXECUTED,
|
|
326
|
+
)
|
|
327
|
+
)
|
|
328
|
+
elif "error" in steps:
|
|
329
|
+
checks.append(mk(ops_id, ops_name, CheckStatus.FAIL, steps["error"]["detail"], Evidence.EXECUTED))
|
|
330
|
+
else:
|
|
331
|
+
checks.append(mk(ops_id, ops_name, CheckStatus.SKIPPED, "not reached: GPU not available"))
|
|
332
|
+
return checks
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""ROCm, HIP, driver, library, tool and PyTorch discovery.
|
|
2
|
+
|
|
3
|
+
Static detection only: nothing here runs GPU code. Runtime execution lives in
|
|
4
|
+
:mod:`kernelvet.diagnose.runtime`.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
|
|
13
|
+
from kernelvet.models.system import SoftwareInfo, ToolInfo, TorchInfo
|
|
14
|
+
from kernelvet.utils.proc import HostContext
|
|
15
|
+
from kernelvet.utils.versions import version_str
|
|
16
|
+
|
|
17
|
+
# (tool, version argv or None)
|
|
18
|
+
TOOLS: list[tuple[str, list[str] | None]] = [
|
|
19
|
+
("rocminfo", None),
|
|
20
|
+
("amd-smi", ["amd-smi", "version"]),
|
|
21
|
+
("rocm-smi", None),
|
|
22
|
+
("hipcc", ["hipcc", "--version"]),
|
|
23
|
+
("hipconfig", ["hipconfig", "--version"]),
|
|
24
|
+
("hipify-clang", ["hipify-clang", "--version"]),
|
|
25
|
+
("hipify-perl", ["hipify-perl", "--version"]),
|
|
26
|
+
("rocprofv3", ["rocprofv3", "--version"]),
|
|
27
|
+
("rocprof", None),
|
|
28
|
+
("cmake", ["cmake", "--version"]),
|
|
29
|
+
("make", ["make", "--version"]),
|
|
30
|
+
("docker", ["docker", "--version"]),
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
# library family -> shared-object stem
|
|
34
|
+
ROCM_LIBRARIES: dict[str, str] = {
|
|
35
|
+
"HIP runtime": "libamdhip64",
|
|
36
|
+
"rocBLAS": "librocblas",
|
|
37
|
+
"hipBLAS": "libhipblas",
|
|
38
|
+
"hipBLASLt": "libhipblaslt",
|
|
39
|
+
"MIOpen": "libMIOpen",
|
|
40
|
+
"RCCL": "librccl",
|
|
41
|
+
"rocFFT": "librocfft",
|
|
42
|
+
"hipFFT": "libhipfft",
|
|
43
|
+
"rocRAND": "librocrand",
|
|
44
|
+
"hipRAND": "libhiprand",
|
|
45
|
+
"rocSPARSE": "librocsparse",
|
|
46
|
+
"hipSPARSE": "libhipsparse",
|
|
47
|
+
"rocSOLVER": "librocsolver",
|
|
48
|
+
"hipSOLVER": "libhipsolver",
|
|
49
|
+
"MIGraphX": "libmigraphx",
|
|
50
|
+
"rocprofiler-sdk": "librocprofiler-sdk",
|
|
51
|
+
"roctracer": "libroctracer64",
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
HEADER_LIBRARIES: dict[str, str] = {
|
|
55
|
+
"rocThrust": "include/thrust/system/hip",
|
|
56
|
+
"hipCUB": "include/hipcub",
|
|
57
|
+
"rocPRIM": "include/rocprim",
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _first_line(text: str) -> str:
|
|
62
|
+
for line in text.splitlines():
|
|
63
|
+
if line.strip():
|
|
64
|
+
return line.strip()
|
|
65
|
+
return ""
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def detect_tools(ctx: HostContext) -> list[ToolInfo]:
|
|
69
|
+
out: list[ToolInfo] = []
|
|
70
|
+
for name, argv in TOOLS:
|
|
71
|
+
path = ctx.which(name)
|
|
72
|
+
if not path:
|
|
73
|
+
out.append(ToolInfo(name=name, available=False))
|
|
74
|
+
continue
|
|
75
|
+
version = None
|
|
76
|
+
detail = None
|
|
77
|
+
if argv:
|
|
78
|
+
res = ctx.run(argv, timeout=min(ctx.timeout, 20))
|
|
79
|
+
text = res.stdout + "\n" + res.stderr
|
|
80
|
+
if res.ok or text.strip():
|
|
81
|
+
version = _tool_version(name, text)
|
|
82
|
+
detail = _first_line(text)[:200] or None
|
|
83
|
+
else:
|
|
84
|
+
detail = res.describe_failure()
|
|
85
|
+
out.append(ToolInfo(name=name, available=True, path=path, version=version, detail=detail))
|
|
86
|
+
return out
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _tool_version(name: str, text: str) -> str | None:
|
|
90
|
+
if name == "hipcc":
|
|
91
|
+
m = re.search(r"HIP version:\s*(\S+)", text)
|
|
92
|
+
return m.group(1) if m else version_str(text)
|
|
93
|
+
if name == "hipconfig":
|
|
94
|
+
first = _first_line(text).split()
|
|
95
|
+
return first[0] if first and version_str(first[0]) else version_str(text)
|
|
96
|
+
if name == "amd-smi":
|
|
97
|
+
m = re.search(r"amdsmi Library version:\s*([\d.]+)", text) or re.search(
|
|
98
|
+
r"AMDSMI Tool:\s*([\d.+\w-]+)", text
|
|
99
|
+
)
|
|
100
|
+
return m.group(1) if m else version_str(text)
|
|
101
|
+
return version_str(text)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def detect_rocm_version(ctx: HostContext, rocm_path: str) -> str | None:
|
|
105
|
+
for rel in (".info/version", ".info/version-dev", ".info/version-utils"):
|
|
106
|
+
text = ctx.read_text(f"{rocm_path}/{rel}")
|
|
107
|
+
if text and text.strip():
|
|
108
|
+
return text.strip().splitlines()[0]
|
|
109
|
+
# /opt/rocm-6.2.0 style install directories
|
|
110
|
+
for name in reversed(ctx.listdir("/opt")):
|
|
111
|
+
m = re.fullmatch(r"rocm-(\d+\.\d+(?:\.\d+)?)", name)
|
|
112
|
+
if m:
|
|
113
|
+
return m.group(1)
|
|
114
|
+
return None
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def detect_hip_compiler(ctx: HostContext, tools: list[ToolInfo]) -> tuple[str | None, str | None]:
|
|
118
|
+
"""Return (HIP runtime version, compiler description) from hipconfig/hipcc."""
|
|
119
|
+
runtime = None
|
|
120
|
+
compiler = None
|
|
121
|
+
hipconfig = next((t for t in tools if t.name == "hipconfig" and t.available), None)
|
|
122
|
+
if hipconfig and hipconfig.version:
|
|
123
|
+
runtime = hipconfig.version
|
|
124
|
+
hipcc = next((t for t in tools if t.name == "hipcc" and t.available), None)
|
|
125
|
+
if hipcc:
|
|
126
|
+
res = ctx.run(["hipcc", "--version"], timeout=min(ctx.timeout, 20))
|
|
127
|
+
text = res.stdout + res.stderr
|
|
128
|
+
m = re.search(r"HIP version:\s*(\S+)", text)
|
|
129
|
+
if m and not runtime:
|
|
130
|
+
runtime = m.group(1)
|
|
131
|
+
clang = re.search(r"(AMD clang version [^\n]+|clang version [^\n]+)", text)
|
|
132
|
+
compiler = clang.group(1).strip() if clang else hipcc.version
|
|
133
|
+
return runtime, compiler
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def detect_driver(ctx: HostContext) -> tuple[str | None, bool | None]:
|
|
137
|
+
if ctx.system != "Linux":
|
|
138
|
+
return None, None
|
|
139
|
+
loaded = ctx.exists("/sys/module/amdgpu")
|
|
140
|
+
version = (ctx.read_text("/sys/module/amdgpu/version") or "").strip() or None
|
|
141
|
+
return version, loaded
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def detect_libraries(ctx: HostContext, rocm_path: str) -> list[str]:
|
|
145
|
+
found: list[str] = []
|
|
146
|
+
libdirs = [f"{rocm_path}/lib", f"{rocm_path}/lib64"]
|
|
147
|
+
names = {n for d in libdirs for n in ctx.listdir(d)}
|
|
148
|
+
for family, stem in ROCM_LIBRARIES.items():
|
|
149
|
+
if any(n == f"{stem}.so" or n.startswith(f"{stem}.so") for n in names):
|
|
150
|
+
found.append(family)
|
|
151
|
+
for family, rel in HEADER_LIBRARIES.items():
|
|
152
|
+
if ctx.exists(f"{rocm_path}/{rel}"):
|
|
153
|
+
found.append(family)
|
|
154
|
+
return found
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
TORCH_PROBE = r"""
|
|
158
|
+
import json, sys
|
|
159
|
+
out = {"installed": False, "python": sys.version.split()[0]}
|
|
160
|
+
try:
|
|
161
|
+
import torch
|
|
162
|
+
except Exception as exc: # ImportError or a broken install
|
|
163
|
+
out["error"] = f"{type(exc).__name__}: {exc}"
|
|
164
|
+
print(json.dumps(out)); sys.exit(0)
|
|
165
|
+
out["installed"] = True
|
|
166
|
+
out["version"] = torch.__version__
|
|
167
|
+
out["hip_version"] = getattr(torch.version, "hip", None)
|
|
168
|
+
out["cuda_version"] = getattr(torch.version, "cuda", None)
|
|
169
|
+
print(json.dumps(out))
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def detect_torch(ctx: HostContext, python: str | None = None) -> TorchInfo:
|
|
174
|
+
"""Inspect PyTorch in a child interpreter so a broken install can't crash us.
|
|
175
|
+
|
|
176
|
+
Only reads version metadata; does not initialise the GPU runtime.
|
|
177
|
+
"""
|
|
178
|
+
exe = python or sys.executable
|
|
179
|
+
res = ctx.run([exe, "-c", TORCH_PROBE], timeout=max(ctx.timeout, 60))
|
|
180
|
+
if not res.ok:
|
|
181
|
+
return TorchInfo(installed=False, error=res.describe_failure())
|
|
182
|
+
try:
|
|
183
|
+
data = json.loads(res.stdout.strip().splitlines()[-1])
|
|
184
|
+
except (json.JSONDecodeError, IndexError):
|
|
185
|
+
return TorchInfo(installed=False, error="unexpected output from PyTorch probe")
|
|
186
|
+
return TorchInfo(**data)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def collect_software(ctx: HostContext, *, python: str | None = None, torch: bool = True) -> SoftwareInfo:
|
|
190
|
+
rocm_path = ctx.rocm_path()
|
|
191
|
+
rocm_present = ctx.exists(rocm_path)
|
|
192
|
+
tools = detect_tools(ctx)
|
|
193
|
+
runtime, compiler = detect_hip_compiler(ctx, tools)
|
|
194
|
+
driver, loaded = detect_driver(ctx)
|
|
195
|
+
return SoftwareInfo(
|
|
196
|
+
rocm_version=detect_rocm_version(ctx, rocm_path) if rocm_present else None,
|
|
197
|
+
rocm_path=rocm_path if rocm_present else None,
|
|
198
|
+
hip_runtime_version=runtime,
|
|
199
|
+
hip_compiler_version=compiler,
|
|
200
|
+
amdgpu_driver_version=driver,
|
|
201
|
+
amdgpu_module_loaded=loaded,
|
|
202
|
+
tools=tools,
|
|
203
|
+
rocm_libraries=detect_libraries(ctx, rocm_path) if rocm_present else [],
|
|
204
|
+
torch=detect_torch(ctx, python) if torch else None,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def torch_summary(t: TorchInfo | None) -> str:
|
|
209
|
+
if t is None:
|
|
210
|
+
return "not checked"
|
|
211
|
+
if not t.installed:
|
|
212
|
+
return "not installed"
|
|
213
|
+
if t.hip_version:
|
|
214
|
+
return f"{t.version} (ROCm/HIP {t.hip_version})"
|
|
215
|
+
if t.cuda_version:
|
|
216
|
+
return f"{t.version} (CUDA {t.cuda_version} build, not ROCm)"
|
|
217
|
+
return f"{t.version} (CPU-only build)"
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def dump(info: SoftwareInfo) -> str: # pragma: no cover - debugging helper
|
|
221
|
+
return json.dumps(info.model_dump(), indent=2)
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Operating system, kernel and architecture discovery."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import platform
|
|
7
|
+
import socket
|
|
8
|
+
|
|
9
|
+
from kernelvet.models.system import HostInfo
|
|
10
|
+
from kernelvet.utils.proc import HostContext
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def parse_os_release(text: str) -> dict[str, str]:
|
|
14
|
+
out: dict[str, str] = {}
|
|
15
|
+
for line in text.splitlines():
|
|
16
|
+
line = line.strip()
|
|
17
|
+
if not line or line.startswith("#") or "=" not in line:
|
|
18
|
+
continue
|
|
19
|
+
key, _, value = line.partition("=")
|
|
20
|
+
value = value.strip()
|
|
21
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
22
|
+
value = value[1:-1]
|
|
23
|
+
out[key.strip()] = value
|
|
24
|
+
return out
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _in_container(ctx: HostContext) -> bool:
|
|
28
|
+
if ctx.exists("/.dockerenv") or ctx.exists("/run/.containerenv"):
|
|
29
|
+
return True
|
|
30
|
+
cgroup = ctx.read_text("/proc/1/cgroup") or ""
|
|
31
|
+
return any(tag in cgroup for tag in ("docker", "containerd", "kubepods", "libpod"))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _hostname_hash() -> str:
|
|
35
|
+
try:
|
|
36
|
+
name = socket.gethostname()
|
|
37
|
+
except OSError:
|
|
38
|
+
return "unknown"
|
|
39
|
+
return hashlib.sha256(name.encode()).hexdigest()[:12]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def collect_host(ctx: HostContext) -> HostInfo:
|
|
43
|
+
os_name = ctx.system
|
|
44
|
+
os_id = os_version = os_pretty = None
|
|
45
|
+
if ctx.system == "Linux":
|
|
46
|
+
rel = ctx.read_text("/etc/os-release") or ctx.read_text("/usr/lib/os-release")
|
|
47
|
+
if rel:
|
|
48
|
+
info = parse_os_release(rel)
|
|
49
|
+
os_id = info.get("ID")
|
|
50
|
+
os_version = info.get("VERSION_ID")
|
|
51
|
+
os_pretty = info.get("PRETTY_NAME")
|
|
52
|
+
os_name = info.get("NAME", "Linux")
|
|
53
|
+
elif ctx.system == "Darwin":
|
|
54
|
+
os_name, os_pretty = "macOS", f"macOS {platform.mac_ver()[0] or ''}".strip()
|
|
55
|
+
elif ctx.system == "Windows":
|
|
56
|
+
os_name, os_pretty = "Windows", f"Windows {platform.release()}"
|
|
57
|
+
return HostInfo(
|
|
58
|
+
os_name=os_name,
|
|
59
|
+
os_id=os_id,
|
|
60
|
+
os_version=os_version,
|
|
61
|
+
os_pretty=os_pretty,
|
|
62
|
+
kernel=ctx.release or None,
|
|
63
|
+
architecture=ctx.machine or "unknown",
|
|
64
|
+
hostname_hash=_hostname_hash(),
|
|
65
|
+
python_version=platform.python_version(),
|
|
66
|
+
in_container=ctx.system == "Linux" and _in_container(ctx),
|
|
67
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|