stack-guardian 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stack_guardian/__init__.py +13 -0
- stack_guardian/__main__.py +6 -0
- stack_guardian/checks.d/10_core.py +174 -0
- stack_guardian/cli.py +295 -0
- stack_guardian/collect.py +206 -0
- stack_guardian/core.py +264 -0
- stack_guardian/finding.py +491 -0
- stack_guardian/fixes.py +287 -0
- stack_guardian/mcp_server.py +340 -0
- stack_guardian/multinode.py +152 -0
- stack_guardian/parsers.py +658 -0
- stack_guardian/serve.py +198 -0
- stack_guardian/state.py +266 -0
- stack_guardian-0.1.0.dist-info/METADATA +164 -0
- stack_guardian-0.1.0.dist-info/RECORD +19 -0
- stack_guardian-0.1.0.dist-info/WHEEL +5 -0
- stack_guardian-0.1.0.dist-info/entry_points.txt +4 -0
- stack_guardian-0.1.0.dist-info/licenses/LICENSE +21 -0
- stack_guardian-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,658 @@
|
|
|
1
|
+
"""sg_parsers -- pure parsers for the "stack-guardian" snapshot.
|
|
2
|
+
|
|
3
|
+
Standard library only. No shelling out, no network: the caller passes the raw
|
|
4
|
+
command / HTTP output text in and gets plain data structures back.
|
|
5
|
+
|
|
6
|
+
Public API
|
|
7
|
+
----------
|
|
8
|
+
parse_nvsmi(gpu_csv: str, apps_csv: str = "") -> list
|
|
9
|
+
parse_tailscale(status_json: str, peer_host: str) -> dict
|
|
10
|
+
parse_ollama_ps(ps_json: str) -> list
|
|
11
|
+
|
|
12
|
+
CLI
|
|
13
|
+
---
|
|
14
|
+
python sg_parsers.py --nvsmi GPU_CSV_FILE [--apps APPS_CSV_FILE]
|
|
15
|
+
python sg_parsers.py --selftest
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import contextlib
|
|
19
|
+
import io
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import shutil
|
|
23
|
+
import sys
|
|
24
|
+
import tempfile
|
|
25
|
+
|
|
26
|
+
__all__ = ["parse_nvsmi", "parse_tailscale", "parse_ollama_ps", "main"]
|
|
27
|
+
|
|
28
|
+
_UNKNOWN = {"peer": False, "direct": None, "derp": None, "ips": []}
|
|
29
|
+
UNAVAILABLE = {"peer": None, "direct": None, "derp": None, "ips": []}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _basename(path: str) -> str:
|
|
33
|
+
"""Basename of a process path that may use either Windows or POSIX separators."""
|
|
34
|
+
return path.replace("\\", "/").rsplit("/", 1)[-1]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _to_int(field: str):
|
|
38
|
+
"""int(field) or None when the field is not an int (e.g. "[N/A]")."""
|
|
39
|
+
try:
|
|
40
|
+
return int(field)
|
|
41
|
+
except (TypeError, ValueError):
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _parse_gpu_lines(gpu_csv: str) -> list:
|
|
46
|
+
"""One dict per parsable GPU line, in input order. Never raises."""
|
|
47
|
+
gpus = []
|
|
48
|
+
for line in (gpu_csv or "").splitlines():
|
|
49
|
+
line = line.strip()
|
|
50
|
+
if not line:
|
|
51
|
+
continue
|
|
52
|
+
try:
|
|
53
|
+
index, uuid, name, total_mib, used_mib = [f.strip() for f in line.split(",", 4)]
|
|
54
|
+
idx = int(index)
|
|
55
|
+
total = int(total_mib)
|
|
56
|
+
used = int(used_mib)
|
|
57
|
+
except (AttributeError, ValueError):
|
|
58
|
+
continue
|
|
59
|
+
# attributed_mib / unattributed_mib are recomputed per-GPU in
|
|
60
|
+
# parse_nvsmi once the compute-app lines are attached.
|
|
61
|
+
gpus.append({
|
|
62
|
+
"index": idx,
|
|
63
|
+
"uuid": uuid,
|
|
64
|
+
"name": name,
|
|
65
|
+
"total_mib": total,
|
|
66
|
+
"used_mib": used,
|
|
67
|
+
"attributed_mib": 0,
|
|
68
|
+
"unattributed_mib": 0,
|
|
69
|
+
"processes": [],
|
|
70
|
+
})
|
|
71
|
+
return gpus
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _parse_app_lines(apps_csv: str) -> list:
|
|
75
|
+
"""One dict per parsable compute-app line, in input order. Never raises."""
|
|
76
|
+
apps = []
|
|
77
|
+
for line in (apps_csv or "").splitlines():
|
|
78
|
+
line = line.strip()
|
|
79
|
+
if not line:
|
|
80
|
+
continue
|
|
81
|
+
try:
|
|
82
|
+
# A process path may itself contain commas, so only the memory
|
|
83
|
+
# column is split off from the right.
|
|
84
|
+
uuid, pid, rest = line.split(",", 2)
|
|
85
|
+
pid_int = int(pid.strip())
|
|
86
|
+
name, mem = rest.rsplit(",", 1)
|
|
87
|
+
except (AttributeError, ValueError):
|
|
88
|
+
continue
|
|
89
|
+
apps.append({
|
|
90
|
+
"uuid": uuid.strip(),
|
|
91
|
+
"pid": pid_int,
|
|
92
|
+
"name": _basename(name.strip()),
|
|
93
|
+
"used_mib": _to_int(mem.strip()),
|
|
94
|
+
})
|
|
95
|
+
return apps
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def parse_nvsmi(gpu_csv: str, apps_csv: str = "") -> list:
|
|
99
|
+
"""Parse `nvidia-smi` GPU and compute-app CSV into snapshot GPU records.
|
|
100
|
+
|
|
101
|
+
Lines whose int fields do not parse are skipped; nothing here raises.
|
|
102
|
+
"""
|
|
103
|
+
gpus = _parse_gpu_lines(gpu_csv)
|
|
104
|
+
apps = _parse_app_lines(apps_csv)
|
|
105
|
+
|
|
106
|
+
for gpu in gpus:
|
|
107
|
+
attributed = 0
|
|
108
|
+
processes = []
|
|
109
|
+
for app in apps:
|
|
110
|
+
if app["uuid"] != gpu["uuid"]:
|
|
111
|
+
continue # a process whose uuid matches no GPU is dropped
|
|
112
|
+
processes.append({
|
|
113
|
+
"pid": app["pid"],
|
|
114
|
+
"name": app["name"],
|
|
115
|
+
"used_mib": app["used_mib"],
|
|
116
|
+
})
|
|
117
|
+
# A "[N/A]" memory counts 0 toward attributed_mib.
|
|
118
|
+
if app["used_mib"] is not None:
|
|
119
|
+
attributed += app["used_mib"]
|
|
120
|
+
gpu["attributed_mib"] = attributed
|
|
121
|
+
# Every process memory "[N/A]" (Windows WDDM): what is unattributed cannot be told, so say None.
|
|
122
|
+
all_na = bool(processes) and all(p["used_mib"] is None for p in processes)
|
|
123
|
+
gpu["unattributed_mib"] = None if all_na else max(0, gpu["used_mib"] - attributed)
|
|
124
|
+
gpu["processes"] = processes
|
|
125
|
+
|
|
126
|
+
return gpus
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def parse_tailscale(status_json: str, peer_host: str) -> dict:
|
|
130
|
+
"""Parse `tailscale status --json` for one peer, matched on HostName (case-insensitive)."""
|
|
131
|
+
try:
|
|
132
|
+
data = json.loads(status_json)
|
|
133
|
+
except (TypeError, ValueError):
|
|
134
|
+
return dict(UNAVAILABLE, error="tailscale status is not JSON")
|
|
135
|
+
|
|
136
|
+
if not isinstance(data, dict):
|
|
137
|
+
return dict(UNAVAILABLE, error="tailscale status is not a JSON object")
|
|
138
|
+
backend = data.get("BackendState")
|
|
139
|
+
if isinstance(backend, str) and backend != "Running":
|
|
140
|
+
return dict(UNAVAILABLE, error="tailscaled is %s" % backend)
|
|
141
|
+
peers = data.get("Peer")
|
|
142
|
+
if not isinstance(peers, dict):
|
|
143
|
+
return dict(_UNKNOWN)
|
|
144
|
+
|
|
145
|
+
wanted = peer_host.lower() if isinstance(peer_host, str) else None
|
|
146
|
+
peer = None
|
|
147
|
+
for candidate in peers.values():
|
|
148
|
+
if not isinstance(candidate, dict):
|
|
149
|
+
continue
|
|
150
|
+
host = candidate.get("HostName")
|
|
151
|
+
if isinstance(host, str) and wanted is not None and host.lower() == wanted:
|
|
152
|
+
peer = candidate
|
|
153
|
+
break
|
|
154
|
+
|
|
155
|
+
if peer is None:
|
|
156
|
+
return dict(_UNKNOWN)
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
"peer": bool(peer.get("Online")),
|
|
160
|
+
"direct": bool(peer.get("CurAddr")),
|
|
161
|
+
"derp": peer.get("Relay") or None,
|
|
162
|
+
"ips": list(peer.get("TailscaleIPs") or []),
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def parse_ollama_ps(ps_json: str) -> list:
|
|
167
|
+
"""Parse the body of Ollama GET /api/ps into model records, in order."""
|
|
168
|
+
try:
|
|
169
|
+
data = json.loads(ps_json)
|
|
170
|
+
except (TypeError, ValueError):
|
|
171
|
+
return []
|
|
172
|
+
|
|
173
|
+
if not isinstance(data, dict):
|
|
174
|
+
return []
|
|
175
|
+
models = data.get("models")
|
|
176
|
+
if not isinstance(models, list):
|
|
177
|
+
return []
|
|
178
|
+
|
|
179
|
+
result = []
|
|
180
|
+
for model in models:
|
|
181
|
+
if not isinstance(model, dict):
|
|
182
|
+
continue
|
|
183
|
+
size = _to_int(model.get("size_vram", 0))
|
|
184
|
+
expires = model.get("expires_at", "")
|
|
185
|
+
name = model.get("name", "")
|
|
186
|
+
result.append({
|
|
187
|
+
"name": "" if name is None else str(name),
|
|
188
|
+
"size_vram": 0 if size is None else size,
|
|
189
|
+
"expires_at": "" if expires is None else str(expires),
|
|
190
|
+
})
|
|
191
|
+
return result
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _cli_error(message: str) -> int:
|
|
195
|
+
sys.stderr.write("sg_parsers: " + message + "\n")
|
|
196
|
+
return 1
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _split_cli_argv(argv: list):
|
|
200
|
+
"""Accept `--flag VALUE`, `--flag=VALUE` and flags in any order."""
|
|
201
|
+
nvsmi_file = None
|
|
202
|
+
apps_file = None
|
|
203
|
+
i = 0
|
|
204
|
+
while i < len(argv):
|
|
205
|
+
token = argv[i]
|
|
206
|
+
key, sep, inline = token.partition("=")
|
|
207
|
+
if key == "--nvsmi":
|
|
208
|
+
nvsmi_file = inline if sep else _cli_value(argv, i)
|
|
209
|
+
i += 2
|
|
210
|
+
elif key == "--apps":
|
|
211
|
+
apps_file = inline if sep else _cli_value(argv, i)
|
|
212
|
+
i += 2
|
|
213
|
+
elif key == "--selftest":
|
|
214
|
+
return "selftest", None
|
|
215
|
+
else:
|
|
216
|
+
return None, "unrecognized argument: " + token
|
|
217
|
+
if nvsmi_file is None:
|
|
218
|
+
return None, "--nvsmi GPU_CSV_FILE is required"
|
|
219
|
+
return (nvsmi_file, apps_file), None
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _cli_value(argv: list, i: int):
|
|
223
|
+
return argv[i + 1] if i + 1 < len(argv) else ""
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _read_text(path: str) -> str:
|
|
227
|
+
with open(path, "r", encoding="utf-8", errors="replace") as handle:
|
|
228
|
+
return handle.read()
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def main(argv=None) -> int:
|
|
232
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
233
|
+
if not argv:
|
|
234
|
+
sys.stderr.write("usage: python sg_parsers.py --nvsmi GPU_CSV_FILE [--apps APPS_CSV_FILE]\n")
|
|
235
|
+
sys.stderr.write(" python sg_parsers.py --selftest\n")
|
|
236
|
+
return 1
|
|
237
|
+
|
|
238
|
+
parsed, err = _split_cli_argv(argv)
|
|
239
|
+
if err:
|
|
240
|
+
return _cli_error(err)
|
|
241
|
+
if parsed == "selftest":
|
|
242
|
+
return _run_selftest()
|
|
243
|
+
|
|
244
|
+
nvsmi_file, apps_file = parsed
|
|
245
|
+
try:
|
|
246
|
+
gpu_csv = _read_text(nvsmi_file)
|
|
247
|
+
apps_csv = _read_text(apps_file) if apps_file else ""
|
|
248
|
+
except OSError as exc:
|
|
249
|
+
return _cli_error("cannot read input: " + str(exc))
|
|
250
|
+
|
|
251
|
+
sys.stdout.write(json.dumps(parse_nvsmi(gpu_csv, apps_csv)) + "\n")
|
|
252
|
+
return 0
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
# --------------------------------------------------------------------------
|
|
256
|
+
# selftest
|
|
257
|
+
# --------------------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
def _check_nvsmi_basic() -> bool:
|
|
260
|
+
"""No apps output still yields correct unattributed_mib for every GPU."""
|
|
261
|
+
res = parse_nvsmi("0, GPU-9658, NVIDIA GeForce RTX 3060, 12288, 0\n"
|
|
262
|
+
"1, GPU-e138, NVIDIA GeForce RTX 3070, 8192, 4096")
|
|
263
|
+
ok = len(res) == 2
|
|
264
|
+
ok = ok and res[0]["index"] == 0 and res[0]["uuid"] == "GPU-9658"
|
|
265
|
+
ok = ok and res[0]["name"] == "NVIDIA GeForce RTX 3060"
|
|
266
|
+
ok = ok and res[0]["total_mib"] == 12288 and res[0]["used_mib"] == 0
|
|
267
|
+
ok = ok and res[0]["attributed_mib"] == 0 and res[0]["unattributed_mib"] == 0
|
|
268
|
+
ok = ok and res[0]["processes"] == []
|
|
269
|
+
ok = ok and res[1]["index"] == 1 and res[1]["total_mib"] == 8192
|
|
270
|
+
ok = ok and res[1]["used_mib"] == 4096 and res[1]["attributed_mib"] == 0
|
|
271
|
+
# The contract wants max(0, used_mib - attributed_mib) even with no apps.
|
|
272
|
+
ok = ok and res[1]["unattributed_mib"] == 4096 and res[1]["processes"] == []
|
|
273
|
+
return ok
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _check_nvsmi_with_apps() -> bool:
|
|
277
|
+
"""Apps memory is attributed per-GPU and split off the remainder."""
|
|
278
|
+
res = parse_nvsmi("0, GPU-9658, NVIDIA GeForce RTX 3060, 12288, 1024",
|
|
279
|
+
"GPU-9658, 15808, C:\\Windows\\explorer.exe, 512\n"
|
|
280
|
+
"GPU-9658, 15809, C:\\Program Files\\Chrome\\chrome.exe, 256")
|
|
281
|
+
ok = len(res) == 1
|
|
282
|
+
ok = ok and res[0]["attributed_mib"] == 768 and res[0]["unattributed_mib"] == 256
|
|
283
|
+
procs = res[0]["processes"]
|
|
284
|
+
ok = ok and len(procs) == 2
|
|
285
|
+
ok = ok and procs[0] == {"pid": 15808, "name": "explorer.exe", "used_mib": 512}
|
|
286
|
+
ok = ok and procs[1] == {"pid": 15809, "name": "chrome.exe", "used_mib": 256}
|
|
287
|
+
return ok
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _check_nvsmi_app_na_memory() -> bool:
|
|
291
|
+
"""'[N/A]' used_memory becomes None and contributes 0 to attributed_mib."""
|
|
292
|
+
res = parse_nvsmi("0, GPU-9658, NVIDIA GeForce RTX 3060, 12288, 1400",
|
|
293
|
+
"GPU-9658, 15808, C:\\Windows\\explorer.exe, [N/A]\n"
|
|
294
|
+
"GPU-9658, 15809, C:\\Program Files\\tool, 700\n"
|
|
295
|
+
"GPU-9658, 15810, /x/junk, abc")
|
|
296
|
+
ok = len(res) == 1
|
|
297
|
+
ok = ok and res[0]["attributed_mib"] == 700 and res[0]["unattributed_mib"] == 700
|
|
298
|
+
procs = res[0]["processes"]
|
|
299
|
+
ok = ok and len(procs) == 3
|
|
300
|
+
ok = ok and procs[0] == {"pid": 15808, "name": "explorer.exe", "used_mib": None}
|
|
301
|
+
ok = ok and procs[1] == {"pid": 15809, "name": "tool", "used_mib": 700}
|
|
302
|
+
# Anything non-int, not just the literal "[N/A]", becomes None.
|
|
303
|
+
ok = ok and procs[2] == {"pid": 15810, "name": "junk", "used_mib": None}
|
|
304
|
+
return ok
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _check_nvsmi_path_with_commas() -> bool:
|
|
308
|
+
"""A path containing commas survives the split(",", 2) / rsplit(",", 1) rules."""
|
|
309
|
+
res = parse_nvsmi("0, GPU-9658, NVIDIA GeForce RTX 3060, 12288, 100",
|
|
310
|
+
"GPU-9658, 4242, /opt/models/a,b/run, 100")
|
|
311
|
+
ok = len(res) == 1 and res[0]["attributed_mib"] == 100
|
|
312
|
+
procs = res[0]["processes"]
|
|
313
|
+
ok = ok and len(procs) == 1 and procs[0]["pid"] == 4242
|
|
314
|
+
ok = ok and procs[0]["name"] == "run" and procs[0]["used_mib"] == 100
|
|
315
|
+
return ok
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def _check_nvsmi_posix_basename() -> bool:
|
|
319
|
+
"""Basename splits on '/' as well as on the Windows separator."""
|
|
320
|
+
res = parse_nvsmi("0, GPU-9658, RTX 3060, 12288, 10",
|
|
321
|
+
"GPU-9658, 7, /opt/vllm/bin/python3, 10")
|
|
322
|
+
ok = len(res) == 1 and len(res[0]["processes"]) == 1
|
|
323
|
+
ok = ok and res[0]["processes"][0]["name"] == "python3"
|
|
324
|
+
return ok
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _check_nvsmi_unknown_uuid_dropped() -> bool:
|
|
328
|
+
"""A process whose uuid matches no GPU is dropped."""
|
|
329
|
+
res = parse_nvsmi("0, GPU-9658, RTX 3060, 12288, 300",
|
|
330
|
+
"GPU-zzzz, 99, /bin/ghost, 300")
|
|
331
|
+
ok = len(res) == 1
|
|
332
|
+
ok = ok and res[0]["processes"] == []
|
|
333
|
+
ok = ok and res[0]["attributed_mib"] == 0 and res[0]["unattributed_mib"] == 300
|
|
334
|
+
return ok
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _check_nvsmi_skips_bad_lines() -> bool:
|
|
338
|
+
"""Blank lines and lines whose int fields do not parse are skipped, never raised on."""
|
|
339
|
+
res = parse_nvsmi("0, GPU-9658, RTX 3060, 12288, 5\n"
|
|
340
|
+
"\n"
|
|
341
|
+
" \n"
|
|
342
|
+
"invalid_line\n"
|
|
343
|
+
"1, GPU-e138, RTX 3070, invalid, 4096\n"
|
|
344
|
+
"1, GPU-e138, RTX 3070, 8192, 4096")
|
|
345
|
+
ok = len(res) == 2
|
|
346
|
+
ok = ok and [g["uuid"] for g in res] == ["GPU-9658", "GPU-e138"]
|
|
347
|
+
ok = ok and res[0]["unattributed_mib"] == 5
|
|
348
|
+
return ok
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _check_nvsmi_multi_gpu_order() -> bool:
|
|
352
|
+
"""GPU order is input order and attribution is per uuid."""
|
|
353
|
+
res = parse_nvsmi("2, GPU-c, RTX A, 100, 10\n"
|
|
354
|
+
"0, GPU-a, RTX B, 200, 20\n"
|
|
355
|
+
"1, GPU-b, RTX C, 300, 30",
|
|
356
|
+
"GPU-b, 1, /x/b.exe, 5\n"
|
|
357
|
+
"GPU-a, 2, /x/a.exe, 7\n"
|
|
358
|
+
"GPU-b, 3, /x/c.exe, [N/A]")
|
|
359
|
+
ok = len(res) == 3
|
|
360
|
+
ok = ok and [g["index"] for g in res] == [2, 0, 1]
|
|
361
|
+
ok = ok and [g["uuid"] for g in res] == ["GPU-c", "GPU-a", "GPU-b"]
|
|
362
|
+
ok = ok and res[0]["processes"] == [] and res[0]["unattributed_mib"] == 10
|
|
363
|
+
ok = ok and res[1]["attributed_mib"] == 7 and res[1]["unattributed_mib"] == 13
|
|
364
|
+
ok = ok and res[2]["attributed_mib"] == 5 and res[2]["unattributed_mib"] == 25
|
|
365
|
+
ok = ok and [p["pid"] for p in res[2]["processes"]] == [1, 3]
|
|
366
|
+
return ok
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _check_nvsmi_attribution_exceeds_used() -> bool:
|
|
370
|
+
"""unattributed_mib is clamped at 0 when processes report more than used."""
|
|
371
|
+
res = parse_nvsmi("0, GPU-9658, RTX 3060, 12288, 10",
|
|
372
|
+
"GPU-9658, 1, /x/a.exe, 60\n"
|
|
373
|
+
"GPU-9658, 2, /x/b.exe, 40")
|
|
374
|
+
ok = len(res) == 1
|
|
375
|
+
ok = ok and res[0]["attributed_mib"] == 100 and res[0]["unattributed_mib"] == 0
|
|
376
|
+
return ok
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _check_nvsmi_never_raises() -> bool:
|
|
380
|
+
"""No malformed input shape may raise, however the int columns are broken."""
|
|
381
|
+
ok = True
|
|
382
|
+
for junk in [
|
|
383
|
+
"not a csv line at all",
|
|
384
|
+
"x, GPU-a, Name, 100, 5", # non-int index
|
|
385
|
+
"0, GPU-a, Name, 100", # too few columns
|
|
386
|
+
"0, GPU-a, Name, 100, 5, extra", # extra columns -> int() must fail, not split oddly
|
|
387
|
+
",,,,", # all empty
|
|
388
|
+
"0, GPU-a, a, b, c, 100, 5", # commas inside the name column
|
|
389
|
+
"0, GPU-a, Name, 1.5, 5", # float memory
|
|
390
|
+
"0, GPU-a, Name, 100, [N/A]",
|
|
391
|
+
]:
|
|
392
|
+
try:
|
|
393
|
+
res = parse_nvsmi(junk)
|
|
394
|
+
except Exception: # noqa: BLE001 -- the contract says never raise
|
|
395
|
+
ok = False
|
|
396
|
+
continue
|
|
397
|
+
if not isinstance(res, list):
|
|
398
|
+
ok = False
|
|
399
|
+
for gpu in res:
|
|
400
|
+
if gpu["unattributed_mib"] != max(0, gpu["used_mib"] - gpu["attributed_mib"]):
|
|
401
|
+
ok = False
|
|
402
|
+
# The same guarantee applies to the apps blob, including a bad pid.
|
|
403
|
+
for junk_apps in ["nope", "GPU-a, notapid, /x/a.exe, 5", "GPU-a, 1", ",,,"]:
|
|
404
|
+
try:
|
|
405
|
+
parse_nvsmi("0, GPU-a, Name, 100, 5", junk_apps)
|
|
406
|
+
except Exception: # noqa: BLE001
|
|
407
|
+
ok = False
|
|
408
|
+
# Exactly one line survives the mixed bag, with its unattributed value intact.
|
|
409
|
+
try:
|
|
410
|
+
mixed = parse_nvsmi("x, GPU-a, Name, 100, 5\n0, GPU-b, Name, 100, 5")
|
|
411
|
+
except Exception: # noqa: BLE001
|
|
412
|
+
mixed = None
|
|
413
|
+
return ok and mixed == [{"index": 0, "uuid": "GPU-b", "name": "Name", "total_mib": 100,
|
|
414
|
+
"used_mib": 5, "attributed_mib": 0, "unattributed_mib": 5,
|
|
415
|
+
"processes": []}]
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _check_nvsmi_empty() -> bool:
|
|
419
|
+
"""Empty and whitespace-only input give an empty list, with an optional apps blob."""
|
|
420
|
+
ok = parse_nvsmi("") == []
|
|
421
|
+
ok = ok and parse_nvsmi(" \n\n") == []
|
|
422
|
+
ok = ok and parse_nvsmi("", "GPU-9658, 1, /x/a.exe, 60") == []
|
|
423
|
+
return ok
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _check_tailscale_online_direct() -> bool:
|
|
427
|
+
"""An online peer with a CurAddr is direct and reports its derp and IPs."""
|
|
428
|
+
status = json.dumps({"Peer": {"p1": {
|
|
429
|
+
"HostName": "myhost", "Online": True, "CurAddr": "10.0.0.1",
|
|
430
|
+
"Relay": "us-central1",
|
|
431
|
+
"TailscaleIPs": ["100.64.0.1", "fd7a:115c:a1e0:ab12:3456:7890:abcd:ef12"]}}})
|
|
432
|
+
res = parse_tailscale(status, "myhost")
|
|
433
|
+
ok = res["peer"] is True and res["direct"] is True
|
|
434
|
+
ok = ok and res["derp"] == "us-central1"
|
|
435
|
+
ok = ok and res["ips"] == ["100.64.0.1", "fd7a:115c:a1e0:ab12:3456:7890:abcd:ef12"]
|
|
436
|
+
ok = ok and len(res) == 4
|
|
437
|
+
return ok
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _check_tailscale_offline_derp() -> bool:
|
|
441
|
+
"""An offline peer behind a relay: peer/direct False, derp kept."""
|
|
442
|
+
status = json.dumps({"Peer": {"p1": {
|
|
443
|
+
"HostName": "box", "Online": False, "CurAddr": "",
|
|
444
|
+
"Relay": "sea", "TailscaleIPs": []}}})
|
|
445
|
+
res = parse_tailscale(status, "box")
|
|
446
|
+
ok = res["peer"] is False and res["direct"] is False
|
|
447
|
+
ok = ok and res["derp"] == "sea" and res["ips"] == []
|
|
448
|
+
return ok
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _check_tailscale_empty_relay_is_none() -> bool:
|
|
452
|
+
"""A falsy Relay (empty string, as tailscale reports a direct connection) is None."""
|
|
453
|
+
status = json.dumps({"Peer": {"p1": {
|
|
454
|
+
"HostName": "direct-box", "Online": True, "CurAddr": "10.1.1.9",
|
|
455
|
+
"Relay": "", "TailscaleIPs": ["100.64.0.9"]}}})
|
|
456
|
+
res = parse_tailscale(status, "direct-box")
|
|
457
|
+
ok = res == {"peer": True, "direct": True, "derp": None, "ips": ["100.64.0.9"]}
|
|
458
|
+
ok = ok and res["derp"] is None
|
|
459
|
+
return ok
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _check_tailscale_case_insensitive() -> bool:
|
|
463
|
+
"""HostName matching is case-insensitive on both sides."""
|
|
464
|
+
status = json.dumps({"Peer": {
|
|
465
|
+
"p1": {"HostName": "Other", "Online": True},
|
|
466
|
+
"p2": {"HostName": "MyBox", "Online": True, "CurAddr": "192.168.1.5"}}})
|
|
467
|
+
res = parse_tailscale(status, "mybox")
|
|
468
|
+
ok = res["peer"] is True and res["direct"] is True
|
|
469
|
+
ok = ok and res["derp"] is None and res["ips"] == []
|
|
470
|
+
return ok
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _check_tailscale_missing_relay_keys() -> bool:
|
|
474
|
+
"""Absent optional keys do not raise; Online/CurAddr fall back to False."""
|
|
475
|
+
status = json.dumps({"Peer": {"p1": {"HostName": "bare"}}})
|
|
476
|
+
res = parse_tailscale(status, "bare")
|
|
477
|
+
ok = res == {"peer": False, "direct": False, "derp": None, "ips": []}
|
|
478
|
+
return ok
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def _check_tailscale_not_found() -> bool:
|
|
482
|
+
"""A peer that is absent gives the not-found record with direct None."""
|
|
483
|
+
status = json.dumps({"Peer": {"p1": {"HostName": "otherhost", "Online": True}}})
|
|
484
|
+
return parse_tailscale(status, "myhost") == dict(_UNKNOWN)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _check_tailscale_invalid_json() -> bool:
|
|
488
|
+
"""Unparsable JSON gives peer None plus an error rather than raising or claiming the peer is offline."""
|
|
489
|
+
ok = parse_tailscale('{"invalid": json}', "myhost")["peer"] is None
|
|
490
|
+
ok = ok and parse_tailscale("", "myhost")["peer"] is None
|
|
491
|
+
ok = ok and parse_tailscale("not json at all", "myhost")["error"] == "tailscale status is not JSON"
|
|
492
|
+
ok = ok and parse_tailscale(json.dumps({"BackendState": "NeedsLogin"}), "myhost")["error"] == "tailscaled is NeedsLogin"
|
|
493
|
+
return ok
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _check_tailscale_bad_shape() -> bool:
|
|
497
|
+
"""A non-dict Peer, a non-dict peer entry, or a JSON list gives not-found."""
|
|
498
|
+
ok = parse_tailscale(json.dumps({"Peer": ["myhost"]}), "myhost") == dict(_UNKNOWN)
|
|
499
|
+
ok = ok and parse_tailscale(json.dumps({"Peer": {"p1": "myhost"}}), "myhost") == dict(_UNKNOWN)
|
|
500
|
+
ok = ok and parse_tailscale(json.dumps([{"HostName": "myhost"}]), "myhost")["peer"] is None
|
|
501
|
+
ok = ok and parse_tailscale(json.dumps({"Peer": {"p1": {"HostName": 7}}}), "myhost") == dict(_UNKNOWN)
|
|
502
|
+
return ok
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _check_ollama_ps_basic() -> bool:
|
|
506
|
+
"""Models come back in order with int size_vram and str expires_at."""
|
|
507
|
+
body = json.dumps({"models": [
|
|
508
|
+
{"name": "llama3:latest", "size_vram": 1024, "expires_at": "2026-01-01T00:00:00Z"},
|
|
509
|
+
{"name": "mistral", "size_vram": 2048, "expires_at": "2026-01-02T00:00:00Z"}]})
|
|
510
|
+
res = parse_ollama_ps(body)
|
|
511
|
+
ok = len(res) == 2
|
|
512
|
+
ok = ok and res[0] == {"name": "llama3:latest", "size_vram": 1024,
|
|
513
|
+
"expires_at": "2026-01-01T00:00:00Z"}
|
|
514
|
+
ok = ok and res[1] == {"name": "mistral", "size_vram": 2048,
|
|
515
|
+
"expires_at": "2026-01-02T00:00:00Z"}
|
|
516
|
+
return ok
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _check_ollama_ps_defaults() -> bool:
|
|
520
|
+
"""Missing size_vram is 0 and missing expires_at is the empty string."""
|
|
521
|
+
res = parse_ollama_ps(json.dumps({"models": [{"name": "phi3"}]}))
|
|
522
|
+
ok = res == [{"name": "phi3", "size_vram": 0, "expires_at": ""}]
|
|
523
|
+
ok = ok and parse_ollama_ps(json.dumps({"models": [{}]})) == [
|
|
524
|
+
{"name": "", "size_vram": 0, "expires_at": ""}]
|
|
525
|
+
return ok
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _check_ollama_ps_bad_values() -> bool:
|
|
529
|
+
"""Non-int size_vram becomes 0 and non-dict entries are skipped."""
|
|
530
|
+
res = parse_ollama_ps(json.dumps({"models": [
|
|
531
|
+
{"name": "a", "size_vram": None, "expires_at": None},
|
|
532
|
+
"not a dict",
|
|
533
|
+
{"name": "b", "size_vram": "128", "expires_at": "soon"}]}))
|
|
534
|
+
ok = len(res) == 2
|
|
535
|
+
ok = ok and res[0] == {"name": "a", "size_vram": 0, "expires_at": ""}
|
|
536
|
+
ok = ok and res[1] == {"name": "b", "size_vram": 128, "expires_at": "soon"}
|
|
537
|
+
return ok
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
def _check_ollama_ps_invalid_json() -> bool:
|
|
541
|
+
"""Unparsable JSON gives an empty list."""
|
|
542
|
+
ok = parse_ollama_ps('{"invalid": json}') == []
|
|
543
|
+
ok = ok and parse_ollama_ps("") == []
|
|
544
|
+
ok = ok and parse_ollama_ps("[]") == []
|
|
545
|
+
return ok
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _check_ollama_ps_missing_models() -> bool:
|
|
549
|
+
"""No 'models' key, or one that is not a list, gives an empty list."""
|
|
550
|
+
ok = parse_ollama_ps(json.dumps({"other": "data"})) == []
|
|
551
|
+
ok = ok and parse_ollama_ps(json.dumps({"models": "none"})) == []
|
|
552
|
+
ok = ok and parse_ollama_ps(json.dumps({"models": []})) == []
|
|
553
|
+
return ok
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _safe_json_loads(text: str):
|
|
557
|
+
"""json.loads(text) or None, so a check can never die on malformed output."""
|
|
558
|
+
try:
|
|
559
|
+
return json.loads(text)
|
|
560
|
+
except (TypeError, ValueError):
|
|
561
|
+
return None
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def _check_cli_nvsmi_one_line_json() -> bool:
|
|
565
|
+
"""--nvsmi prints json.dumps(result) as exactly one line of valid JSON."""
|
|
566
|
+
gpu_csv = "0, GPU-9658, NVIDIA GeForce RTX 3060, 12288, 1024"
|
|
567
|
+
apps_csv = "GPU-9658, 15808, C:\\Windows\\explorer.exe, 512"
|
|
568
|
+
workdir = tempfile.mkdtemp()
|
|
569
|
+
ok = False
|
|
570
|
+
try:
|
|
571
|
+
gpu_path = os.path.join(workdir, "gpu.csv")
|
|
572
|
+
apps_path = os.path.join(workdir, "apps.csv")
|
|
573
|
+
with open(gpu_path, "w", encoding="utf-8") as handle:
|
|
574
|
+
handle.write(gpu_csv)
|
|
575
|
+
with open(apps_path, "w", encoding="utf-8") as handle:
|
|
576
|
+
handle.write(apps_csv)
|
|
577
|
+
|
|
578
|
+
buf = io.StringIO()
|
|
579
|
+
with contextlib.redirect_stdout(buf):
|
|
580
|
+
code = main(["--nvsmi", gpu_path, "--apps", apps_path])
|
|
581
|
+
printed = buf.getvalue()
|
|
582
|
+
lines = printed.splitlines()
|
|
583
|
+
payload = _safe_json_loads(lines[0]) if len(lines) == 1 else None
|
|
584
|
+
|
|
585
|
+
ok = (code == 0 and printed.count("\n") == 1 and printed.endswith("\n")
|
|
586
|
+
and len(lines) == 1 and payload is not None)
|
|
587
|
+
ok = ok and payload == parse_nvsmi(gpu_csv, apps_csv)
|
|
588
|
+
ok = ok and payload[0]["attributed_mib"] == 512
|
|
589
|
+
ok = ok and payload[0]["unattributed_mib"] == 512
|
|
590
|
+
ok = ok and payload[0]["processes"][0]["name"] == "explorer.exe"
|
|
591
|
+
|
|
592
|
+
# The same run without --apps must still print exactly one JSON line.
|
|
593
|
+
buf2 = io.StringIO()
|
|
594
|
+
with contextlib.redirect_stdout(buf2):
|
|
595
|
+
code2 = main(["--nvsmi", gpu_path])
|
|
596
|
+
printed2 = buf2.getvalue()
|
|
597
|
+
payload2 = _safe_json_loads(printed2)
|
|
598
|
+
ok = ok and code2 == 0 and printed2.count("\n") == 1
|
|
599
|
+
ok = ok and payload2 is not None and payload2[0]["unattributed_mib"] == 1024
|
|
600
|
+
finally:
|
|
601
|
+
shutil.rmtree(workdir, ignore_errors=True)
|
|
602
|
+
return ok
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
SELFTESTS = [
|
|
606
|
+
("nvsmi_basic", _check_nvsmi_basic),
|
|
607
|
+
("nvsmi_with_apps", _check_nvsmi_with_apps),
|
|
608
|
+
("nvsmi_app_na_memory", _check_nvsmi_app_na_memory),
|
|
609
|
+
("nvsmi_path_with_commas", _check_nvsmi_path_with_commas),
|
|
610
|
+
("nvsmi_posix_basename", _check_nvsmi_posix_basename),
|
|
611
|
+
("nvsmi_unknown_uuid_dropped", _check_nvsmi_unknown_uuid_dropped),
|
|
612
|
+
("nvsmi_skips_bad_lines", _check_nvsmi_skips_bad_lines),
|
|
613
|
+
("nvsmi_multi_gpu_order", _check_nvsmi_multi_gpu_order),
|
|
614
|
+
("nvsmi_attribution_exceeds_used", _check_nvsmi_attribution_exceeds_used),
|
|
615
|
+
("nvsmi_never_raises", _check_nvsmi_never_raises),
|
|
616
|
+
("nvsmi_empty", _check_nvsmi_empty),
|
|
617
|
+
("tailscale_online_direct", _check_tailscale_online_direct),
|
|
618
|
+
("tailscale_offline_derp", _check_tailscale_offline_derp),
|
|
619
|
+
("tailscale_empty_relay_is_none", _check_tailscale_empty_relay_is_none),
|
|
620
|
+
("tailscale_case_insensitive", _check_tailscale_case_insensitive),
|
|
621
|
+
("tailscale_missing_relay_keys", _check_tailscale_missing_relay_keys),
|
|
622
|
+
("tailscale_not_found", _check_tailscale_not_found),
|
|
623
|
+
("tailscale_invalid_json", _check_tailscale_invalid_json),
|
|
624
|
+
("tailscale_bad_shape", _check_tailscale_bad_shape),
|
|
625
|
+
("ollama_ps_basic", _check_ollama_ps_basic),
|
|
626
|
+
("ollama_ps_defaults", _check_ollama_ps_defaults),
|
|
627
|
+
("ollama_ps_bad_values", _check_ollama_ps_bad_values),
|
|
628
|
+
("ollama_ps_invalid_json", _check_ollama_ps_invalid_json),
|
|
629
|
+
("ollama_ps_missing_models", _check_ollama_ps_missing_models),
|
|
630
|
+
("cli_nvsmi_one_line_json", _check_cli_nvsmi_one_line_json),
|
|
631
|
+
]
|
|
632
|
+
|
|
633
|
+
|
|
634
|
+
def _run_selftest() -> int:
|
|
635
|
+
"""Run every named check; the pass counter is incremented from each return value."""
|
|
636
|
+
passed = 0
|
|
637
|
+
for name, check in SELFTESTS:
|
|
638
|
+
try:
|
|
639
|
+
result = check()
|
|
640
|
+
except Exception as exc: # a raising check is a failing check, never a crash
|
|
641
|
+
result = False
|
|
642
|
+
sys.stderr.write("sg_parsers selftest: " + name + " raised "
|
|
643
|
+
+ type(exc).__name__ + ": " + str(exc) + "\n")
|
|
644
|
+
if result:
|
|
645
|
+
passed += 1
|
|
646
|
+
else:
|
|
647
|
+
sys.stderr.write("sg_parsers selftest: FAIL " + name + "\n")
|
|
648
|
+
|
|
649
|
+
total = len(SELFTESTS)
|
|
650
|
+
failed = total - passed
|
|
651
|
+
sys.stdout.write("sg_parsers selftest: " + str(total) + " checks, "
|
|
652
|
+
+ str(passed) + " passed, " + str(failed) + " failed\n")
|
|
653
|
+
sys.stdout.flush()
|
|
654
|
+
return 0 if failed == 0 and total > 0 else 1
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
if __name__ == "__main__":
|
|
658
|
+
sys.exit(main())
|