gputaskman 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {gputaskman-0.3.2 → gputaskman-0.3.3}/PKG-INFO +36 -3
- {gputaskman-0.3.2 → gputaskman-0.3.3}/README.md +35 -2
- {gputaskman-0.3.2 → gputaskman-0.3.3}/pyproject.toml +1 -1
- {gputaskman-0.3.2 → gputaskman-0.3.3}/pyproject.toml.orig +1 -1
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/app.py +124 -37
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/cli.py +3 -3
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/collector.py +3 -1
- {gputaskman-0.3.2 → gputaskman-0.3.3}/LICENSE +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/__init__.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/__main__.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/adl.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/amdsmi.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/diff.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/drm.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/dxgi.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/gguf.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/handy.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/linux.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/llm.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/macos.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/nvml.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/pdh.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/procinfo.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/report.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/stores.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/telemetry.py +0 -0
- {gputaskman-0.3.2 → gputaskman-0.3.3}/src/gputaskman/wslhost.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: gputaskman
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: An htop for what's inside your GPU: per-process VRAM, engine load, and LLM weights / KV cache breakdown
|
|
5
5
|
Author: Scott Hanselman
|
|
6
6
|
License-Expression: MIT
|
|
@@ -44,12 +44,33 @@ CPU-only, NPU-only, and cloud workloads are out of scope.
|
|
|
44
44
|
> Early, experimental software. Device and process counters are measurements; classifications
|
|
45
45
|
> and GGUF-derived breakdowns are estimates. This does not dump VRAM or enumerate a game's textures.
|
|
46
46
|
|
|
47
|
+
## Install from PyPI
|
|
48
|
+
|
|
49
|
+
[gputaskman is on PyPI](https://pypi.org/project/gputaskman/). No repository clone is needed:
|
|
50
|
+
|
|
51
|
+
```powershell
|
|
52
|
+
uv tool install gputaskman
|
|
53
|
+
gputaskman
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Or, with [pipx](https://pipx.pypa.io/):
|
|
57
|
+
|
|
58
|
+
```powershell
|
|
59
|
+
pipx install gputaskman
|
|
60
|
+
gputaskman
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Works on Windows, Linux/WSL, and macOS with Python 3.11+ and the supported GPU/driver
|
|
64
|
+
configuration described below. Both installers keep the app in an isolated environment.
|
|
65
|
+
The shorter `gtm` command is also available. Upgrade with `uv tool upgrade gputaskman`
|
|
66
|
+
or `pipx upgrade gputaskman`. For Windows without Python, use the portable download below.
|
|
67
|
+
|
|
47
68
|
## Portable Windows download
|
|
48
69
|
|
|
49
70
|
The [Releases page](https://github.com/shanselman/gputaskman/releases) is where tagged builds
|
|
50
71
|
publish two portable ZIPs: `gputaskman-VERSION-windows-x64.zip` for Intel/AMD Windows PCs, and
|
|
51
72
|
`gputaskman-VERSION-windows-arm64.zip` for native Windows ARM64. These are separate native builds,
|
|
52
|
-
not a universal executable.
|
|
73
|
+
not a universal executable.
|
|
53
74
|
|
|
54
75
|
Extract the **entire ZIP**, keeping `_internal` beside `gputaskman.exe`. Open Windows Terminal
|
|
55
76
|
or PowerShell in the extracted folder:
|
|
@@ -421,12 +442,24 @@ scale, temperature uses 0–120°C, and the power and clock scales are displayed
|
|
|
421
442
|
The dashboard and inspector scroll independently in short terminals so the process list stays usable.
|
|
422
443
|
Model/runtime discovery is throttled, so newly loaded models may take about 15 seconds to appear.
|
|
423
444
|
|
|
445
|
+
On multi-GPU systems, the TUI stacks a header and trend charts for each GPU and adds a **GPU**
|
|
446
|
+
column to the combined process list. A process using two GPUs has a separate row for each;
|
|
447
|
+
selecting a row inspects its memory on that GPU. Single-GPU layouts are unchanged.
|
|
448
|
+
Memory totals and histories stay separate per device; model files and runtime buffer logs
|
|
449
|
+
can span devices and do not prove per-device model placement. Discovery's 150 ms scan budget
|
|
450
|
+
is divided across GPUs, though each GPU has its own counter and telemetry queries.
|
|
451
|
+
This layout is covered by synthetic tests, **not yet validated on multi-GPU hardware**.
|
|
452
|
+
Under multi-GPU WSL, host attribution is disabled with a reason rather than risking matching
|
|
453
|
+
two devices by memory usage; Linux processes are listed once with GPU `?`, because `/dev/dxg`
|
|
454
|
+
does not identify their GPU. `--gpu N` retains the existing single-device host matching.
|
|
455
|
+
|
|
424
456
|
`--list-gpus` lists selectable hardware adapters with their vendor and LUID. `--gpu N` selects
|
|
425
457
|
one of these adapters: NVIDIA adapters are listed first, followed by other vendors, with each
|
|
426
458
|
group ordered by dedicated memory. An out-of-range
|
|
427
459
|
index is rejected rather than silently selecting another device. Compute-only devices such as an
|
|
428
460
|
NPU are not graphics adapters. AMD and NVIDIA hardware coverage is described below; configurations
|
|
429
|
-
with multiple GPUs and mixed vendors need further validation.
|
|
461
|
+
with multiple GPUs and mixed vendors need further hardware validation. `--once` and `--json`
|
|
462
|
+
still default to GPU 0; use `--gpu N` to collect another device without changing the snapshot format.
|
|
430
463
|
Device telemetry is matched to WDDM adapters by PCI location, not display name or enumeration order.
|
|
431
464
|
If a unique match cannot be established, telemetry is unavailable; buffer logs can still span devices,
|
|
432
465
|
so model-buffer attribution across GPUs is not yet reliable. This conservative matching also applies
|
|
@@ -22,12 +22,33 @@ CPU-only, NPU-only, and cloud workloads are out of scope.
|
|
|
22
22
|
> Early, experimental software. Device and process counters are measurements; classifications
|
|
23
23
|
> and GGUF-derived breakdowns are estimates. This does not dump VRAM or enumerate a game's textures.
|
|
24
24
|
|
|
25
|
+
## Install from PyPI
|
|
26
|
+
|
|
27
|
+
[gputaskman is on PyPI](https://pypi.org/project/gputaskman/). No repository clone is needed:
|
|
28
|
+
|
|
29
|
+
```powershell
|
|
30
|
+
uv tool install gputaskman
|
|
31
|
+
gputaskman
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
Or, with [pipx](https://pipx.pypa.io/):
|
|
35
|
+
|
|
36
|
+
```powershell
|
|
37
|
+
pipx install gputaskman
|
|
38
|
+
gputaskman
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Works on Windows, Linux/WSL, and macOS with Python 3.11+ and the supported GPU/driver
|
|
42
|
+
configuration described below. Both installers keep the app in an isolated environment.
|
|
43
|
+
The shorter `gtm` command is also available. Upgrade with `uv tool upgrade gputaskman`
|
|
44
|
+
or `pipx upgrade gputaskman`. For Windows without Python, use the portable download below.
|
|
45
|
+
|
|
25
46
|
## Portable Windows download
|
|
26
47
|
|
|
27
48
|
The [Releases page](https://github.com/shanselman/gputaskman/releases) is where tagged builds
|
|
28
49
|
publish two portable ZIPs: `gputaskman-VERSION-windows-x64.zip` for Intel/AMD Windows PCs, and
|
|
29
50
|
`gputaskman-VERSION-windows-arm64.zip` for native Windows ARM64. These are separate native builds,
|
|
30
|
-
not a universal executable.
|
|
51
|
+
not a universal executable.
|
|
31
52
|
|
|
32
53
|
Extract the **entire ZIP**, keeping `_internal` beside `gputaskman.exe`. Open Windows Terminal
|
|
33
54
|
or PowerShell in the extracted folder:
|
|
@@ -399,12 +420,24 @@ scale, temperature uses 0–120°C, and the power and clock scales are displayed
|
|
|
399
420
|
The dashboard and inspector scroll independently in short terminals so the process list stays usable.
|
|
400
421
|
Model/runtime discovery is throttled, so newly loaded models may take about 15 seconds to appear.
|
|
401
422
|
|
|
423
|
+
On multi-GPU systems, the TUI stacks a header and trend charts for each GPU and adds a **GPU**
|
|
424
|
+
column to the combined process list. A process using two GPUs has a separate row for each;
|
|
425
|
+
selecting a row inspects its memory on that GPU. Single-GPU layouts are unchanged.
|
|
426
|
+
Memory totals and histories stay separate per device; model files and runtime buffer logs
|
|
427
|
+
can span devices and do not prove per-device model placement. Discovery's 150 ms scan budget
|
|
428
|
+
is divided across GPUs, though each GPU has its own counter and telemetry queries.
|
|
429
|
+
This layout is covered by synthetic tests, **not yet validated on multi-GPU hardware**.
|
|
430
|
+
Under multi-GPU WSL, host attribution is disabled with a reason rather than risking matching
|
|
431
|
+
two devices by memory usage; Linux processes are listed once with GPU `?`, because `/dev/dxg`
|
|
432
|
+
does not identify their GPU. `--gpu N` retains the existing single-device host matching.
|
|
433
|
+
|
|
402
434
|
`--list-gpus` lists selectable hardware adapters with their vendor and LUID. `--gpu N` selects
|
|
403
435
|
one of these adapters: NVIDIA adapters are listed first, followed by other vendors, with each
|
|
404
436
|
group ordered by dedicated memory. An out-of-range
|
|
405
437
|
index is rejected rather than silently selecting another device. Compute-only devices such as an
|
|
406
438
|
NPU are not graphics adapters. AMD and NVIDIA hardware coverage is described below; configurations
|
|
407
|
-
with multiple GPUs and mixed vendors need further validation.
|
|
439
|
+
with multiple GPUs and mixed vendors need further hardware validation. `--once` and `--json`
|
|
440
|
+
still default to GPU 0; use `--gpu N` to collect another device without changing the snapshot format.
|
|
408
441
|
Device telemetry is matched to WDDM adapters by PCI location, not display name or enumeration order.
|
|
409
442
|
If a unique match cannot be established, telemetry is unavailable; buffer logs can still span devices,
|
|
410
443
|
so model-buffer attribution across GPUs is not yet reliable. This conservative matching also applies
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import os
|
|
6
|
-
import time
|
|
7
6
|
from collections import deque
|
|
7
|
+
from contextlib import ExitStack
|
|
8
8
|
|
|
9
9
|
from rich.console import Group
|
|
10
10
|
from rich.table import Table
|
|
@@ -17,7 +17,7 @@ from textual.screen import ModalScreen
|
|
|
17
17
|
from textual.widgets import DataTable, Footer, Markdown, Select, Static
|
|
18
18
|
from textual.worker import get_current_worker
|
|
19
19
|
|
|
20
|
-
from .collector import BACKEND, Collector, ProcRow, Snapshot
|
|
20
|
+
from .collector import BACKEND, Collector, ProcRow, Snapshot, enumerate_adapters, ordered_adapters
|
|
21
21
|
from .llm import gb
|
|
22
22
|
from .nvml import NvDevice, format_metric
|
|
23
23
|
from .report import (DRM_RESIDENT_NOTE, METAL_NOTE, WSL_EMPTY, drm_note, has_engine_counters, has_spill_counters, has_wddm_counters,
|
|
@@ -174,6 +174,7 @@ class GpuTaskManager(App):
|
|
|
174
174
|
#chart-controls { height: 1; }
|
|
175
175
|
#chart-label { width: 8; text-style: bold; }
|
|
176
176
|
#chart-mode { width: 24; }
|
|
177
|
+
#chart-mode SelectOverlay { border: solid $accent; padding: 0 1; }
|
|
177
178
|
#chart-hint { width: 1fr; color: $text-muted; }
|
|
178
179
|
#charts { height: auto; }
|
|
179
180
|
#chart-note { height: auto; color: $text-muted; }
|
|
@@ -203,12 +204,15 @@ class GpuTaskManager(App):
|
|
|
203
204
|
}
|
|
204
205
|
CHARTS = tuple(CHART_COLORS)
|
|
205
206
|
|
|
206
|
-
def __init__(self, gpu_index: int =
|
|
207
|
+
def __init__(self, gpu_index: int | None = None, interval: float = 1.0, host: bool = True) -> None:
|
|
207
208
|
super().__init__()
|
|
208
209
|
self.gpu_index = gpu_index
|
|
209
210
|
self.interval = interval
|
|
210
211
|
self.host = host
|
|
211
212
|
self.snap: Snapshot | None = None
|
|
213
|
+
self.snapshots: list[Snapshot] = []
|
|
214
|
+
self._histories: dict[int, dict[str, deque]] = {}
|
|
215
|
+
self._row_snapshots: dict[str, Snapshot] = {}
|
|
212
216
|
self.selected_key: str | None = None # ProcRow.key: Linux and Windows host PIDs can collide
|
|
213
217
|
self.sort_idx = 0
|
|
214
218
|
self.show_all = False
|
|
@@ -228,7 +232,8 @@ class GpuTaskManager(App):
|
|
|
228
232
|
def selected_pid(self) -> int | None:
|
|
229
233
|
"""PID of the selected local process; None for Windows (W:) and outside (H:) rows."""
|
|
230
234
|
key = self.selected_key
|
|
231
|
-
|
|
235
|
+
pid = key.rsplit(":", 1)[-1] if key else ""
|
|
236
|
+
return int(pid) if pid.isdigit() else None
|
|
232
237
|
|
|
233
238
|
@selected_pid.setter
|
|
234
239
|
def selected_pid(self, pid: int | None) -> None:
|
|
@@ -262,31 +267,70 @@ class GpuTaskManager(App):
|
|
|
262
267
|
@work(thread=True, exclusive=True)
|
|
263
268
|
def poll(self) -> None:
|
|
264
269
|
worker = get_current_worker()
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
270
|
+
indices = ([self.gpu_index] if self.gpu_index is not None else
|
|
271
|
+
list(range(len(ordered_adapters(enumerate_adapters())))) or [0])
|
|
272
|
+
with ExitStack() as stack:
|
|
273
|
+
collectors = []
|
|
274
|
+
for index in indices:
|
|
275
|
+
collector = Collector(index, scan_budget=Collector.SCAN_BUDGET / len(indices),
|
|
276
|
+
host=self.host and not (BACKEND == "wsl" and len(indices) > 1))
|
|
277
|
+
if BACKEND == "wsl" and len(indices) > 1 and self.host:
|
|
278
|
+
collector.host_status = "off (multi-GPU WSL host matching is not validated; use --gpu N)"
|
|
279
|
+
stack.callback(collector.close)
|
|
280
|
+
collectors.append(collector)
|
|
281
|
+
worker.cancelled_event.wait(0.5)
|
|
268
282
|
while not worker.is_cancelled:
|
|
269
283
|
if not self.paused:
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
collector.close()
|
|
284
|
+
snapshots = [collector.sample() for collector in collectors]
|
|
285
|
+
if not worker.is_cancelled:
|
|
286
|
+
self.call_from_thread(self.apply_all, snapshots)
|
|
287
|
+
worker.cancelled_event.wait(self.interval)
|
|
275
288
|
|
|
276
289
|
# ------------------------------------------------------------------ rendering
|
|
277
290
|
|
|
278
291
|
def apply(self, snap: Snapshot) -> None:
|
|
292
|
+
self.apply_all([snap])
|
|
293
|
+
|
|
294
|
+
def apply_all(self, snapshots: list[Snapshot]) -> None:
|
|
295
|
+
if not self.is_running:
|
|
296
|
+
return # A worker may finish sampling after shutdown has removed the widgets.
|
|
297
|
+
self.snapshots = snapshots
|
|
298
|
+
multi = len(snapshots) > 1
|
|
299
|
+
table = self.query_one("#procs", DataTable)
|
|
300
|
+
if multi and "GPU" not in table.columns:
|
|
301
|
+
table.add_column("GPU", key="GPU")
|
|
302
|
+
elif not multi and "GPU" in table.columns:
|
|
303
|
+
table.remove_column("GPU")
|
|
304
|
+
self._row_snapshots = {}
|
|
305
|
+
for index, snap in enumerate(snapshots):
|
|
306
|
+
for row in snap.procs:
|
|
307
|
+
row.gpu_index = index if multi else None
|
|
308
|
+
self._row_snapshots[row.key] = snap
|
|
309
|
+
self._use_history(index)
|
|
310
|
+
self._record_sample(snap)
|
|
311
|
+
self._use_history(0)
|
|
312
|
+
self.snap = snapshots[0]
|
|
313
|
+
if not self._auto_selected:
|
|
314
|
+
llm = next((r for r in sorted(
|
|
315
|
+
(r for s in snapshots for r in s.procs), key=self.SORTS[0][1]) if r.llm), None)
|
|
316
|
+
if llm:
|
|
317
|
+
self.selected_key = llm.key
|
|
318
|
+
self._auto_selected = True
|
|
319
|
+
self.render_dashboard()
|
|
320
|
+
self.render_table(self.snap)
|
|
321
|
+
self.render_inspector()
|
|
322
|
+
|
|
323
|
+
def _use_history(self, index: int) -> None:
|
|
324
|
+
names = ("util", "clock", "bw", "vram", "temp", "power", "enc", "dec")
|
|
325
|
+
if index not in self._histories:
|
|
326
|
+
self._histories[index] = {name: deque(maxlen=60) for name in names}
|
|
327
|
+
for name, history in self._histories[index].items():
|
|
328
|
+
setattr(self, f"hist_{name}", history)
|
|
329
|
+
|
|
330
|
+
def _record_sample(self, snap: Snapshot) -> None:
|
|
279
331
|
self.snap = snap
|
|
280
332
|
nv = snap.nv
|
|
281
333
|
amd = bool(snap.adapter and snap.adapter.vendor_id == 0x1002)
|
|
282
|
-
if getattr(self, "_amd_chart_labels", None) != amd:
|
|
283
|
-
self._amd_chart_labels = amd
|
|
284
|
-
selector = self.query_one("#chart-mode", Select)
|
|
285
|
-
selected = selector.value
|
|
286
|
-
selector.set_options([("All charts", -1), *(
|
|
287
|
-
("GFX clock" if amd and self.CHARTS[index] == "SM clock" else self.CHARTS[index], index)
|
|
288
|
-
for index in self.available_charts)]) # macOS offers only the charts IOKit has data for
|
|
289
|
-
selector.value = selected
|
|
290
334
|
if nv:
|
|
291
335
|
self.hist_util.append(nv.util_gpu)
|
|
292
336
|
self.hist_clock.append(nv.gfx_clock if amd else nv.sm_clock)
|
|
@@ -301,15 +345,27 @@ class GpuTaskManager(App):
|
|
|
301
345
|
history.append(None)
|
|
302
346
|
self.hist_vram.append(100 * snap.vram_used / snap.vram_total
|
|
303
347
|
if snap.vram_total and snap.vram_used is not None else None)
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
348
|
+
|
|
349
|
+
def render_dashboard(self) -> None:
|
|
350
|
+
amd = bool(self.snapshots[0].adapter and self.snapshots[0].adapter.vendor_id == 0x1002)
|
|
351
|
+
clock_label = "SM/GFX clock" if len(self.snapshots) > 1 else "GFX clock" if amd else "SM clock"
|
|
352
|
+
if getattr(self, "_clock_label", None) != clock_label:
|
|
353
|
+
self._clock_label = clock_label
|
|
354
|
+
selector = self.query_one("#chart-mode", Select)
|
|
355
|
+
selected = selector.value
|
|
356
|
+
selector.set_options([("All charts", -1), *(
|
|
357
|
+
(clock_label if self.CHARTS[index] == "SM clock" else self.CHARTS[index], index)
|
|
358
|
+
for index in self.available_charts)])
|
|
359
|
+
selector.value = selected
|
|
360
|
+
self._amd_chart_labels = amd
|
|
361
|
+
devices = []
|
|
362
|
+
for index, snap in enumerate(self.snapshots):
|
|
363
|
+
if len(self.snapshots) > 1:
|
|
364
|
+
devices.append(Text(f"GPU {index}", style="bold"))
|
|
365
|
+
devices.append(self.render_device(snap))
|
|
366
|
+
self.query_one("#device", Static).update(
|
|
367
|
+
Group(*devices) if len(self.snapshots) > 1 else devices[0])
|
|
310
368
|
self.render_charts()
|
|
311
|
-
self.render_table(snap)
|
|
312
|
-
self.render_inspector()
|
|
313
369
|
|
|
314
370
|
def render_device(self, s: Snapshot) -> Group:
|
|
315
371
|
nv = s.nv
|
|
@@ -415,9 +471,28 @@ class GpuTaskManager(App):
|
|
|
415
471
|
return self.hist_dec, 100, "%", color
|
|
416
472
|
|
|
417
473
|
def render_charts(self) -> None:
|
|
418
|
-
if self.
|
|
474
|
+
if len(self.snapshots) > 1:
|
|
475
|
+
charts = []
|
|
476
|
+
notes = []
|
|
477
|
+
for index, snap in enumerate(self.snapshots):
|
|
478
|
+
self.snap = snap
|
|
479
|
+
self._use_history(index)
|
|
480
|
+
self._amd_chart_labels = bool(snap.adapter and snap.adapter.vendor_id == 0x1002)
|
|
481
|
+
grid, note = self._chart_content()
|
|
482
|
+
charts.extend([Text(f"GPU {index}", style="bold"), grid])
|
|
483
|
+
notes.append(f"GPU {index}: {note}")
|
|
484
|
+
self.query_one("#charts", Static).update(Group(*charts))
|
|
485
|
+
self.query_one("#chart-note", Static).update("\n".join(notes))
|
|
486
|
+
self.snap = self.snapshots[0]
|
|
487
|
+
self._use_history(0)
|
|
488
|
+
self._amd_chart_labels = bool(self.snap.adapter and self.snap.adapter.vendor_id == 0x1002)
|
|
419
489
|
return
|
|
420
|
-
|
|
490
|
+
if self.snap is not None:
|
|
491
|
+
grid, note = self._chart_content()
|
|
492
|
+
self.query_one("#charts", Static).update(grid)
|
|
493
|
+
self.query_one("#chart-note", Static).update(note)
|
|
494
|
+
|
|
495
|
+
def _chart_content(self) -> tuple[Table, str]:
|
|
421
496
|
# Reserve the border, padding, and scrollbar even before child resize events arrive.
|
|
422
497
|
width = max(1, self.size.width - 6)
|
|
423
498
|
indices = list(self.available_charts) if self.chart_idx == -1 else [self.chart_idx]
|
|
@@ -439,19 +514,18 @@ class GpuTaskManager(App):
|
|
|
439
514
|
cells.append(cell)
|
|
440
515
|
for start in range(0, len(cells), columns):
|
|
441
516
|
grid.add_row(*cells[start:start + columns])
|
|
442
|
-
widget.update(grid)
|
|
443
517
|
_, power_scale, _, _ = self.chart_series(self.CHARTS.index("Power"), self.snap.nv)
|
|
444
518
|
_, clock_scale, _, _ = self.chart_series(self.CHARTS.index("SM clock"), self.snap.nv)
|
|
445
|
-
|
|
519
|
+
note = (
|
|
446
520
|
"Recent samples · blank = no reading" if self.snap.backend == "metal" else
|
|
447
521
|
f"Recent samples · blank = no reading · power 0–{power_scale:.0f} W · "
|
|
448
522
|
f"{'GFX' if getattr(self, '_amd_chart_labels', False) else 'SM'} clock 0–{clock_scale:.0f} MHz"
|
|
449
523
|
)
|
|
524
|
+
return grid, note
|
|
450
525
|
|
|
451
526
|
def on_resize(self) -> None:
|
|
452
527
|
if self.is_mounted and self.snap is not None:
|
|
453
|
-
self.
|
|
454
|
-
self.call_after_refresh(self.render_charts)
|
|
528
|
+
self.call_after_refresh(self.render_dashboard)
|
|
455
529
|
|
|
456
530
|
def visible_rows(self, s: Snapshot) -> list[ProcRow]:
|
|
457
531
|
rows = s.procs
|
|
@@ -470,7 +544,12 @@ class GpuTaskManager(App):
|
|
|
470
544
|
|
|
471
545
|
def render_table(self, s: Snapshot) -> None:
|
|
472
546
|
t = self.query_one("#procs", DataTable)
|
|
473
|
-
rows = self.visible_rows(s)
|
|
547
|
+
rows = [r for snap in self.snapshots for r in self.visible_rows(snap)] if len(self.snapshots) > 1 else self.visible_rows(s)
|
|
548
|
+
if len(self.snapshots) > 1:
|
|
549
|
+
rows = [r for r in rows if not (
|
|
550
|
+
self._row_snapshots[r.key].backend == "wsl" and not r.host and r.gpu_index != 0)]
|
|
551
|
+
if len(self.snapshots) > 1 and not any(snap.backend == "wsl" for snap in self.snapshots):
|
|
552
|
+
rows.sort(key=self.SORTS[self.sort_idx][1])
|
|
474
553
|
keep = self.selected_key
|
|
475
554
|
t.clear()
|
|
476
555
|
|
|
@@ -480,6 +559,7 @@ class GpuTaskManager(App):
|
|
|
480
559
|
return Text(f"{v:.0f}" if v >= 0.5 else "", justify="right", style="bold" if v >= 20 else "")
|
|
481
560
|
|
|
482
561
|
for r in rows:
|
|
562
|
+
s = self._row_snapshots.get(r.key, s)
|
|
483
563
|
e = r.gpu.engines
|
|
484
564
|
wddm = has_wddm_counters(s, r)
|
|
485
565
|
engines = has_engine_counters(s, r)
|
|
@@ -503,6 +583,8 @@ class GpuTaskManager(App):
|
|
|
503
583
|
pct(e.get("copy", 0) if split else None),
|
|
504
584
|
pct(e.get("decode", 0) + e.get("encode", 0) + e.get("video", 0) if split else None),
|
|
505
585
|
Text(r.inside, style="magenta" if r.llm else "grey62", overflow="ellipsis", no_wrap=True),
|
|
586
|
+
*([Text("?" if s.backend == "wsl" and not r.host else str(r.gpu_index))]
|
|
587
|
+
if len(self.snapshots) > 1 else []),
|
|
506
588
|
key=r.key,
|
|
507
589
|
)
|
|
508
590
|
if keep is not None:
|
|
@@ -516,13 +598,15 @@ class GpuTaskManager(App):
|
|
|
516
598
|
|
|
517
599
|
def render_inspector(self) -> None:
|
|
518
600
|
panel = self.query_one("#inspector", Static)
|
|
519
|
-
s = self.snap
|
|
601
|
+
s = self._row_snapshots.get(self.selected_key, self.snap)
|
|
520
602
|
r = next((x for x in s.procs if x.key == self.selected_key), None) if s else None
|
|
521
603
|
if r is None:
|
|
522
604
|
panel.update(Text("Select a process (↑/↓) to see what's inside its GPU memory. Press ? to learn what's knowable.", style="grey62"))
|
|
523
605
|
return
|
|
524
606
|
where = {"windows": "Windows PID ", "outside": "host PID "}.get(r.origin, "") if r.host else ""
|
|
525
607
|
panel.border_title = f"{r.name} ({where}{r.pid}) - {r.kind}"
|
|
608
|
+
if r.gpu_index is not None:
|
|
609
|
+
panel.border_title += " - GPU unknown" if s.backend == "wsl" and not r.host else f" - GPU {r.gpu_index}"
|
|
526
610
|
out: list = []
|
|
527
611
|
e = r.gpu.engines
|
|
528
612
|
wddm = has_wddm_counters(s, r)
|
|
@@ -571,6 +655,9 @@ class GpuTaskManager(App):
|
|
|
571
655
|
|
|
572
656
|
v = r.llm
|
|
573
657
|
if v:
|
|
658
|
+
if r.gpu_index is not None:
|
|
659
|
+
out.append(Text("Model files, launch flags and runtime buffer logs can span GPUs. "
|
|
660
|
+
"The breakdown below is not a measured per-device model placement.", style="grey70"))
|
|
574
661
|
width = max(min(self.size.width - 8, 100), 20)
|
|
575
662
|
total = max(r.gpu.shown_vram or 0, sum(x.bytes for x in v.segments), 1)
|
|
576
663
|
out.append(stacked_bar([(x.bytes, SEG_STYLE[x.kind]) for x in v.segments], total, width))
|
|
@@ -669,7 +756,7 @@ class GpuTaskManager(App):
|
|
|
669
756
|
def action_pause(self) -> None:
|
|
670
757
|
self.paused = not self.paused
|
|
671
758
|
if self.snap:
|
|
672
|
-
self.
|
|
759
|
+
self.render_dashboard()
|
|
673
760
|
|
|
674
761
|
def action_help(self) -> None:
|
|
675
762
|
self.push_screen(HelpScreen())
|
|
@@ -34,13 +34,13 @@ def main() -> None:
|
|
|
34
34
|
epilog="gputaskman diff BEFORE.json AFTER.json compares two --json snapshots.")
|
|
35
35
|
ap.add_argument("--once", action="store_true", help="print one snapshot and exit (no TUI)")
|
|
36
36
|
ap.add_argument("--json", action="store_true", help="print one JSON snapshot and exit (implies --once)")
|
|
37
|
-
ap.add_argument("--gpu", type=int,
|
|
37
|
+
ap.add_argument("--gpu", type=int, help="show only this GPU (TUI default: all; one-shot default: 0)")
|
|
38
38
|
ap.add_argument("--list-gpus", action="store_true", help="list selectable GPU adapters and exit")
|
|
39
39
|
ap.add_argument("--no-host", action="store_true",
|
|
40
40
|
help="WSL: don't read Windows host GPU counters through interop (WSL VM total, Windows apps)")
|
|
41
41
|
ap.add_argument("-i", "--interval", type=float, default=1.0, help="refresh seconds (default 1)")
|
|
42
42
|
args = ap.parse_args()
|
|
43
|
-
if args.gpu < 0:
|
|
43
|
+
if args.gpu is not None and args.gpu < 0:
|
|
44
44
|
ap.error("--gpu must be a non-negative index")
|
|
45
45
|
if not math.isfinite(args.interval) or args.interval <= 0:
|
|
46
46
|
ap.error("--interval must be a finite number greater than zero")
|
|
@@ -68,7 +68,7 @@ def main() -> None:
|
|
|
68
68
|
from .report import dump
|
|
69
69
|
|
|
70
70
|
try:
|
|
71
|
-
c = Collector(args.gpu, scan_budget=None, host=not args.no_host) # one-shot: identify everything now
|
|
71
|
+
c = Collector(args.gpu or 0, scan_budget=None, host=not args.no_host) # one-shot: identify everything now
|
|
72
72
|
except ValueError as error:
|
|
73
73
|
ap.error(str(error))
|
|
74
74
|
try:
|
|
@@ -51,11 +51,13 @@ class ProcRow:
|
|
|
51
51
|
handy: HandyEvidence | None = None
|
|
52
52
|
host: bool = False # pid is from another PID namespace, never a local one (see origin)
|
|
53
53
|
origin: str = "" # with host: "windows" (a Windows process seen from WSL) | "outside" (outside this container)
|
|
54
|
+
gpu_index: int | None = None
|
|
54
55
|
|
|
55
56
|
@property
|
|
56
57
|
def key(self) -> str:
|
|
57
58
|
"""Unique row id: PIDs from other namespaces and local PIDs can collide."""
|
|
58
|
-
|
|
59
|
+
pid = f"{self.origin[0]}{self.pid}" if self.host else str(self.pid)
|
|
60
|
+
return f"{self.gpu_index}:{pid}" if self.gpu_index is not None else pid
|
|
59
61
|
|
|
60
62
|
@property
|
|
61
63
|
def pid_label(self) -> str:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|