gputaskman 0.3.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
gputaskman/app.py ADDED
@@ -0,0 +1,675 @@
1
+ """The htop-style Textual UI."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import os
6
+ import time
7
+ from collections import deque
8
+
9
+ from rich.console import Group
10
+ from rich.table import Table
11
+ from rich.text import Text
12
+ from textual import work
13
+ from textual.app import App, ComposeResult
14
+ from textual.binding import Binding
15
+ from textual.containers import Horizontal, Vertical, VerticalScroll
16
+ from textual.screen import ModalScreen
17
+ from textual.widgets import DataTable, Footer, Markdown, Select, Static
18
+ from textual.worker import get_current_worker
19
+
20
+ from .collector import BACKEND, Collector, ProcRow, Snapshot
21
+ from .llm import gb
22
+ from .nvml import NvDevice, format_metric
23
+ from .report import (DRM_RESIDENT_NOTE, METAL_NOTE, WSL_EMPTY, drm_note, has_engine_counters, has_spill_counters, has_wddm_counters,
24
+ inferred_note, vram_label, wsl_note)
25
+
26
+ KIND_STYLE = {
27
+ "LLM": "bold magenta", "decision": "bold orchid", "decision?": "orchid",
28
+ "embedding": "plum2", "speech": "light_sky_blue1", "diffusion": "orchid",
29
+ "AI": "medium_purple1", "compute": "orange3", "3D/game": "green3",
30
+ "video": "yellow", "browser": "dodger_blue2", "app": "cyan", "compositor": "grey50",
31
+ "kernel": "grey50", "system": "grey50", "wsl-vm": "bold spring_green2", "external": "grey50",
32
+ }
33
+ SEG_STYLE = {"weights": "magenta", "kv": "dark_orange", "compute": "yellow", "runtime": "grey58", "unknown": "grey42"}
34
+ SPARK = "▁▂▃▄▅▆▇█"
35
+
36
+ EXPLAIN = {
37
+ "wsl-vm": "The WSL virtual machine as Windows sees it. Windows charges all GPU memory used by Linux processes "
38
+ "in every WSL distro (and WSLg) to vmwp.exe, so this is the exact total. The split across Linux "
39
+ "processes is not exposed.",
40
+ "compositor": "Desktop Window Manager: its own back buffers and the composed image for each monitor. "
41
+ "Windows' 'Dedicated Usage' counter shows dwm as huge (it counts every app surface it can see); "
42
+ "this tool prefers Local Usage for attribution. The adapter counter is the total; "
43
+ "per-process samples can overlap or differ in timing.",
44
+ "kernel": "Driver/kernel-owned allocations: paging buffers, display scan-out, firmware, scheduler state.",
45
+ "system": "Session/system process holding display and cursor surfaces.",
46
+ "browser": "Chromium GPU process: page tiles, images, video frames, canvas, WebGL/WebGPU buffers. "
47
+ "Deeper: edge://gpu, or DevTools > Performance with the GPU track.",
48
+ "3D/game": "Textures, meshes, render targets and descriptor heaps. Windows only sees opaque allocations; "
49
+ "per-resource names need a graphics debugger (PIX on Windows, NVIDIA Nsight Graphics).",
50
+ "video": "Decoded/encoded video frames and codec surfaces (NVIDIA NVDEC/ENC)",
51
+ "compute": "Compute allocations are opaque to the OS. Runtime-specific profilers can provide more detail (CUDA: cudaMalloc with Nsight Systems).",
52
+ "diffusion": "Model components are identified from launch arguments; GPU allocations cannot be split by component.",
53
+ "AI": "ML runtime allocations (tensors, workspaces). PyTorch: torch.cuda.memory_summary(); "
54
+ "ONNX Runtime/DirectML pools are opaque from outside.",
55
+ "app": "UI surfaces: swap chains, window back-buffers, cached bitmaps/glyphs.",
56
+ }
57
+
58
+ HELP_MD = """\
59
+ # What can be known about GPU memory?
60
+
61
+ | Layer | Source | Works for |
62
+ |---|---|---|
63
+ | **Device** telemetry | NVML, AMD ADL, or optional AMD SMI; amdgpu sysfs on Linux | availability varies by GPU and driver |
64
+ | **Per-process** VRAM, spill to RAM, per-engine load | Windows WDDM counters (dxgkrnl); DRM fdinfo for amdgpu on Linux | workloads visible to WDDM or DRM |
65
+ | **Which runtime** (CUDA, HIP, Vulkan, D3D12, ggml, ONNX, PyTorch...) | DLLs mapped in the process | everything |
66
+ | **Which model file** | memory-mapped files + command line | llama.cpp, stable-diffusion.cpp, LM Studio, Ollama |
67
+ | **Model name & kind** (LLM, decision, embedding, speech) | GGUF header, Ollama/LM Studio/Hugging Face stores, local server APIs | GGUF family |
68
+ | **Weights / KV cache / compute** split | GGUF header math, llama.cpp/Ollama logs, local server APIs | GGUF/llama.cpp family |
69
+ | Per-texture / per-buffer names | needs a debugger (PIX, Nsight) attached to the app | games, 3D apps |
70
+
71
+ **Evidence** shows how a model was identified: `api` (the runtime says it is loaded), `mapped`
72
+ (Windows shows the file mapped into the process right now), `cmdline` (a launch argument only).
73
+ Two sources, including `api` or `mapped`, is shown as *confirmed*.
74
+
75
+ **Local workloads only:** inference, decision models, vision, speech, training, graphics and video.
76
+ Cloud-hosted models are outside this monitor's scope. DLLs identify runtimes, not models or tasks.
77
+ `decision?` / *likely decision model (name)* is a filename or model-folder hint when a specific
78
+ GGUF kind is missing. It does not confirm the kind, GPU residency, or execution. Installed
79
+ candidates remain unproven; engine activity is measured per process, not per model.
80
+
81
+ **Linux / WSL:** on native Linux, per-process VRAM and SM/encoder/decoder load come from NVML
82
+ (no spill or commit counters exist there, so those columns stay blank). **AMD GPUs on Linux**
83
+ (`amdgpu`) use sysfs for the device and DRM fdinfo (`/proc/<pid>/fdinfo`) per process: VRAM,
84
+ GTT (GPU memory in system RAM, shown as SPILL), and gfx/compute/video engine time (`n/a` for a new client's
85
+ first sample and for ROCm/HIP processes: their compute runs on `/dev/kfd` queues the kernel doesn't time per
86
+ process). VRAM and SPILL are charged to a process only when its DRM clients are provably private: one holder,
87
+ opened through a render node, and no shared buffers. Otherwise they show `n/a` and the bytes stay unattributed;
88
+ the inspector still shows the resident bytes it can reach (shared buffers included, so not added up). Only processes you can inspect are
89
+ listed, and the header says how many it couldn't; run with `sudo` to see other users'. Under **WSL**, Windows
90
+ charges all Linux GPU memory to the VM (`vmwp.exe`), so VRAM and load are device-wide only; rows
91
+ are Linux processes with `/dev/dxg` open and show `n/a` rather than a made-up share. When WSL
92
+ interop is enabled, the Windows host's counters are read through `powershell.exe`: **WSL VM**
93
+ (`vmwp.exe`) is then the exact total of Linux GPU memory, and Windows apps appear as `W:<pid>`
94
+ rows. `--no-host` turns this off. Linux rows get their VRAM by matching the VM's GPU contexts
95
+ (newest first) to Linux processes: `*` when candidates agree on the size, `?` for a heuristic owner.
96
+ Cached foreign sizes only help rank guesses; they never prove ownership across runs.
97
+
98
+ **Why does nvidia-smi say N/A?** GeForce cards on Windows run in *WDDM* mode: Windows, not the
99
+ NVIDIA driver, owns video memory, so NVML can't attribute it to processes. The Windows counters can.
100
+
101
+ **dwm.exe looks huge elsewhere?** Windows' per-process "Dedicated Usage" counts every surface a
102
+ process can *see*, so dwm can show nearly the whole card. VRAM here is what's charged to each
103
+ process ("Local Usage" when available). The adapter counter is the total, not the sum of rows.
104
+
105
+ **SPILL** = GPU-accessible system RAM. This can include pinned host buffers or intentional
106
+ allocations, not just overflow. Large usage under VRAM pressure can indicate fallback;
107
+ the counter alone does not prove it.
108
+
109
+ **PAGED** = max(Total Committed - Local Usage - Non Local Usage, 0). This is an estimate of
110
+ non-resident committed memory, not a measured eviction count or proof of pagefile activity.
111
+
112
+ **Inside** labels are inferred from runtimes and files. Log sizes describe allocated buffers,
113
+ not necessarily current residency. A game's textures cannot be enumerated here.
114
+
115
+ **GPU%** = busiest engine (like Task Manager). On GeForce/WDDM, CUDA kernels are usually reported
116
+ on the **3D** engine. AMD device GPU charts show graphics-engine activity, which is not
117
+ necessarily equivalent to NVML GPU utilization. Vulkan does not expose another process's HIP internals.
118
+
119
+ Keys: `↑/↓` select · `s` sort · `c` chart · `a` show all · `p` pause · `?` this help · `q` quit
120
+
121
+ All eight trend charts are visible by default. They stack in narrow terminals and
122
+ share rows when there is room. Click the **Charts** dropdown to show all charts or
123
+ one metric; `c` cycles the same choices. A blank sample means no reading; zero is
124
+ drawn at the baseline. Percentage charts use 0–100%, temperature uses 0–120°C,
125
+ and the power and SM clock scales are shown below the charts. In short terminals,
126
+ scroll the dashboard or inspector without losing the process list.
127
+ """
128
+
129
+
130
+ def spark(values, width: int = 30, top: float = 100.0) -> str:
131
+ vals = list(values)[-width:]
132
+ return "".join(
133
+ SPARK[min(max(int(v / top * (len(SPARK) - 1) + 0.5), 0), len(SPARK) - 1)]
134
+ if v is not None else " "
135
+ for v in vals
136
+ )
137
+
138
+
139
+ def stacked_bar(parts: list[tuple[float, str]], total: float, width: int) -> Text:
140
+ t = Text()
141
+ used = 0
142
+ for value, style in parts:
143
+ n = int(round(value / total * width)) if total else 0
144
+ n = min(n, width - used)
145
+ if n > 0:
146
+ t.append("█" * n, style=style)
147
+ used += n
148
+ t.append("░" * (width - used), style="grey27")
149
+ return t
150
+
151
+
152
+ class HelpScreen(ModalScreen):
153
+ BINDINGS = [Binding("escape,q,question_mark", "app.pop_screen", "Close")]
154
+ DEFAULT_CSS = """
155
+ HelpScreen { align: center middle; }
156
+ HelpScreen > VerticalScroll { width: 140; max-width: 95%; height: 85%; border: thick $accent; background: $surface; padding: 1 2; }
157
+ """
158
+
159
+ def compose(self) -> ComposeResult:
160
+ with VerticalScroll():
161
+ yield Markdown(HELP_MD)
162
+
163
+
164
+ METAL_CHARTS = ("GPU", "VRAM")
165
+
166
+
167
+ class GpuTaskManager(App):
168
+ TITLE = "gputaskman"
169
+ CSS = """
170
+ Screen { layout: vertical; }
171
+ #dashboard { height: auto; max-height: 50%; }
172
+ #device { height: auto; border: round $accent; padding: 0 1; }
173
+ #history { height: auto; border: round $secondary; padding: 0 1; }
174
+ #chart-controls { height: 1; }
175
+ #chart-label { width: 8; text-style: bold; }
176
+ #chart-mode { width: 24; }
177
+ #chart-hint { width: 1fr; color: $text-muted; }
178
+ #charts { height: auto; }
179
+ #chart-note { height: auto; color: $text-muted; }
180
+ #procs { height: 1fr; min-height: 4; }
181
+ #inspector { height: auto; max-height: 30%; overflow-y: auto; border: round $secondary; padding: 0 1; }
182
+ """
183
+ BINDINGS = [
184
+ Binding("q", "quit", "Quit"),
185
+ Binding("s", "cycle_sort", "Sort"),
186
+ Binding("c", "cycle_chart", "Chart"),
187
+ Binding("a", "toggle_all", "All/Top"),
188
+ Binding("p", "pause", "Pause"),
189
+ Binding("question_mark", "help", "Explain"),
190
+ ]
191
+ SORTS = [("VRAM", lambda r: -(r.gpu.shown_vram or 0)), ("GPU%", lambda r: -r.gpu.gpu_pct),
192
+ ("spill", lambda r: -r.gpu.shared), ("paged", lambda r: -r.gpu.paged_out),
193
+ ("name", lambda r: r.name.lower())]
194
+ CHART_COLORS = {
195
+ "GPU": "#4ade80",
196
+ "SM clock": "#fb923c",
197
+ "VRAM": "#60a5fa",
198
+ "Mem activity": "#a78bfa",
199
+ "Temperature": "#ef4444",
200
+ "Power": "#facc15",
201
+ "Encoder": "#e2e8f0",
202
+ "Decoder": "#22d3ee",
203
+ }
204
+ CHARTS = tuple(CHART_COLORS)
205
+
206
+ def __init__(self, gpu_index: int = 0, interval: float = 1.0, host: bool = True) -> None:
207
+ super().__init__()
208
+ self.gpu_index = gpu_index
209
+ self.interval = interval
210
+ self.host = host
211
+ self.snap: Snapshot | None = None
212
+ self.selected_key: str | None = None # ProcRow.key: Linux and Windows host PIDs can collide
213
+ self.sort_idx = 0
214
+ self.show_all = False
215
+ self.paused = False
216
+ self.hist_util: deque = deque(maxlen=60)
217
+ self.hist_clock: deque = deque(maxlen=60)
218
+ self.hist_bw: deque = deque(maxlen=60)
219
+ self.hist_vram: deque = deque(maxlen=60)
220
+ self.hist_temp: deque = deque(maxlen=60)
221
+ self.hist_power: deque = deque(maxlen=60)
222
+ self.hist_enc: deque = deque(maxlen=60)
223
+ self.hist_dec: deque = deque(maxlen=60)
224
+ self.chart_idx = -1
225
+ self._auto_selected = False
226
+
227
+ @property
228
+ def selected_pid(self) -> int | None:
229
+ """PID of the selected local process; None for Windows (W:) and outside (H:) rows."""
230
+ key = self.selected_key
231
+ return int(key) if key and key.isdigit() else None
232
+
233
+ @selected_pid.setter
234
+ def selected_pid(self, pid: int | None) -> None:
235
+ self.selected_key = None if pid is None else str(pid)
236
+
237
+ def compose(self) -> ComposeResult:
238
+ with VerticalScroll(id="dashboard"):
239
+ yield Static("Sampling GPU counters...", id="device")
240
+ with Vertical(id="history"):
241
+ with Horizontal(id="chart-controls"):
242
+ yield Static("Charts", id="chart-label")
243
+ yield Select(
244
+ [("All charts", -1), *((self.CHARTS[index], index) for index in self.available_charts)],
245
+ value=-1, allow_blank=False, compact=True, id="chart-mode",
246
+ )
247
+ yield Static("Click to choose · c cycles", id="chart-hint")
248
+ yield Static(id="charts")
249
+ yield Static(id="chart-note")
250
+ table = DataTable(id="procs", cursor_type="row", zebra_stripes=True)
251
+ yield table
252
+ yield Static("", id="inspector")
253
+ yield Footer()
254
+
255
+ def on_mount(self) -> None:
256
+ t = self.query_one("#procs", DataTable)
257
+ for col in ("PID", "Process", "Kind", "Runtime", "VRAM", "Spill", "Paged", "GPU%", "3D", "Copy", "Video", "What's inside"):
258
+ t.add_column(col, key=col)
259
+ t.focus()
260
+ self.poll()
261
+
262
+ @work(thread=True, exclusive=True)
263
+ def poll(self) -> None:
264
+ worker = get_current_worker()
265
+ collector = Collector(self.gpu_index, host=self.host)
266
+ time.sleep(0.5)
267
+ try:
268
+ while not worker.is_cancelled:
269
+ if not self.paused:
270
+ snap = collector.sample()
271
+ self.call_from_thread(self.apply, snap)
272
+ time.sleep(self.interval)
273
+ finally:
274
+ collector.close()
275
+
276
+ # ------------------------------------------------------------------ rendering
277
+
278
+ def apply(self, snap: Snapshot) -> None:
279
+ self.snap = snap
280
+ nv = snap.nv
281
+ amd = bool(snap.adapter and snap.adapter.vendor_id == 0x1002)
282
+ if getattr(self, "_amd_chart_labels", None) != amd:
283
+ self._amd_chart_labels = amd
284
+ selector = self.query_one("#chart-mode", Select)
285
+ selected = selector.value
286
+ selector.set_options([("All charts", -1), *(
287
+ ("GFX clock" if amd and self.CHARTS[index] == "SM clock" else self.CHARTS[index], index)
288
+ for index in self.available_charts)]) # macOS offers only the charts IOKit has data for
289
+ selector.value = selected
290
+ if nv:
291
+ self.hist_util.append(nv.util_gpu)
292
+ self.hist_clock.append(nv.gfx_clock if amd else nv.sm_clock)
293
+ self.hist_bw.append(nv.util_mem)
294
+ self.hist_temp.append(nv.temp)
295
+ self.hist_power.append(nv.power_w)
296
+ self.hist_enc.append(nv.enc)
297
+ self.hist_dec.append(nv.dec)
298
+ else:
299
+ for history in (self.hist_util, self.hist_clock, self.hist_bw, self.hist_temp,
300
+ self.hist_power, self.hist_enc, self.hist_dec):
301
+ history.append(None)
302
+ self.hist_vram.append(100 * snap.vram_used / snap.vram_total
303
+ if snap.vram_total and snap.vram_used is not None else None)
304
+ if not self._auto_selected:
305
+ llm = next((r for r in sorted(snap.procs, key=self.SORTS[0][1]) if r.llm), None)
306
+ if llm:
307
+ self.selected_key = llm.key
308
+ self._auto_selected = True
309
+ self.query_one("#device", Static).update(self.render_device(snap))
310
+ self.render_charts()
311
+ self.render_table(snap)
312
+ self.render_inspector()
313
+
314
+ def render_device(self, s: Snapshot) -> Group:
315
+ nv = s.nv
316
+ name = s.adapter.name if s.adapter else (nv.name if nv else "GPU")
317
+ if s.backend == "metal":
318
+ head = Text.assemble((name, "bold"), f" {s.driver} ", ("Metal", "bold yellow"), " unified memory")
319
+ else:
320
+ info = (f" driver {s.driver}" if s.driver else "") + (f" CUDA {s.cuda}" if s.cuda else "")
321
+ head = Text.assemble((name, "bold"), info + " ",
322
+ (nv.driver_model if nv else "", "bold yellow"), f" {nv.pci_bus if nv else ''}")
323
+ if self.paused:
324
+ head.append(" [PAUSED]", style="bold red")
325
+
326
+ # Fit category proportions to the adapter total; row samples need not sum exactly.
327
+ by_kind: dict[str, float] = {}
328
+ for r in s.procs:
329
+ if r.gpu.attributed: # unknown shares stay in "unattributed" instead of showing as 0
330
+ by_kind[r.kind] = by_kind.get(r.kind, 0) + r.gpu.dedicated
331
+ attributed = sum(by_kind.values())
332
+ scale = min(1.0, s.vram_used / attributed) if attributed and s.vram_used is not None else 1.0
333
+ order = sorted(by_kind.items(), key=lambda kv: -kv[1])
334
+ parts = [(v * scale, KIND_STYLE.get(k, "white")) for k, v in order] if s.vram_used is not None else []
335
+ other = max(s.vram_used - attributed * scale, 0) if s.vram_used is not None else 0
336
+ parts.append((other, "grey35"))
337
+ width = max(min(self.size.width - 40, 90), 20)
338
+ vram = Text("GPU mem " if s.backend == "metal" else "VRAM ")
339
+ vram.append_text(stacked_bar(parts, s.vram_total, width))
340
+ used_label = gb(s.vram_used) if s.vram_used is not None else "N/A"
341
+ vram.append(f" {used_label} / {gb(s.vram_total)}", style="bold")
342
+ legend = Text(" ")
343
+ for k, v in order[:7]:
344
+ legend.append("■ ", style=KIND_STYLE.get(k, "white"))
345
+ legend.append(f"{k} {gb(v)} ")
346
+ if other > 32 << 20:
347
+ legend.append("■ ", style="grey35")
348
+ legend.append(f"unattributed {gb(other)} ")
349
+ if s.vram_used is not None:
350
+ legend.append(f"free {gb(max(s.vram_total - s.vram_used, 0))}", style="grey62")
351
+ if s.shared_used is not None and s.shared_used > 64 << 20:
352
+ legend.append(f" shared(RAM) {gb(s.shared_used)}", style="yellow")
353
+ if scale < 1.0:
354
+ legend.append(" [bar normalized; labels are raw]", style="grey62")
355
+ if s.backend == "wsl":
356
+ legend.append("\n " + wsl_note(s), style="grey62")
357
+ if s.backend == "metal":
358
+ legend.append("\n " + METAL_NOTE, style="grey62")
359
+ if s.counter_status:
360
+ legend.append("\n " + s.counter_status, style="grey62")
361
+ if s.process_status:
362
+ legend.append(f"\n DRM process scan: {s.process_status}.", style="grey62")
363
+
364
+ lines = [head, vram, legend]
365
+ if nv and s.backend == "metal":
366
+ lines.append(Text(f"GPU {format_metric(nv.util_gpu, unit='%')}",
367
+ style=f"bold {self.CHART_COLORS['GPU']}"))
368
+ elif nv:
369
+ rx = nv.pcie_rx_kbs / 1024 if nv.pcie_rx_kbs is not None else None
370
+ tx = nv.pcie_tx_kbs / 1024 if nv.pcie_tx_kbs is not None else None
371
+ pstate = f"P{nv.pstate}" if nv.pstate is not None else "P N/A"
372
+ m = Text()
373
+ label = "AMD gfx" if nv.source.startswith("AMD ") else "GPU"
374
+ m.append(f"{label} {format_metric(nv.util_gpu, unit='%')}", style=f"bold {self.CHART_COLORS['GPU']}")
375
+ m.append(f" Mem activity {format_metric(nv.util_mem, unit='%')}", style=self.CHART_COLORS["Mem activity"])
376
+ m.append(f" PCIe↓{format_metric(rx, '.0f'):>5} ↑{format_metric(tx, '.0f'):>4} MB/s")
377
+ if nv.enc or nv.dec:
378
+ m.append(f" enc {format_metric(nv.enc, unit='%')} dec {format_metric(nv.dec, unit='%')}")
379
+ lines.append(m)
380
+ clock_label = (f"gfx {format_metric(nv.gfx_clock)} MHz" if nv.source.startswith("AMD ") else
381
+ f"SM {format_metric(nv.sm_clock)}/{format_metric(nv.sm_clock_max)} MHz")
382
+ power_kind = f" {nv.power_kind}" if nv.power_kind else ""
383
+ st = Text(f"{format_metric(nv.temp, unit='°C')} {format_metric(nv.power_w, '.0f')}/"
384
+ f"{format_metric(nv.power_limit_w, '.0f')} W{power_kind} {pstate} "
385
+ f"{clock_label} "
386
+ f"mem {format_metric(nv.mem_clock)} MHz video {format_metric(nv.video_clock)} MHz "
387
+ f"PCIe gen{format_metric(nv.pcie_gen)} x{format_metric(nv.pcie_width)} "
388
+ + (f"CC {nv.compute_cap or 'N/A'}" if nv.source == "NVML" else nv.source), style="grey70")
389
+ if nv.throttle:
390
+ st.append(" throttle: " + ",".join(nv.throttle), style="yellow")
391
+ lines.append(st)
392
+ return Group(*lines)
393
+
394
+ def chart_series(self, index: int, nv: NvDevice | None) -> tuple[deque, float, str, str]:
395
+ chart = self.CHARTS[index]
396
+ color = self.CHART_COLORS[chart]
397
+ if chart == "GPU":
398
+ return self.hist_util, 100, "%", color
399
+ if chart == "SM clock":
400
+ values = [value for value in self.hist_clock if value is not None]
401
+ scale = max((nv.sm_clock_max if nv else None) or 0, max(values, default=0), 1)
402
+ return self.hist_clock, scale, " MHz", color
403
+ if chart == "VRAM":
404
+ return self.hist_vram, 100, "%", color
405
+ if chart == "Mem activity":
406
+ return self.hist_bw, 100, "%", color
407
+ if chart == "Temperature":
408
+ return self.hist_temp, 120, "°C", color
409
+ if chart == "Power":
410
+ values = [value for value in self.hist_power if value is not None]
411
+ scale = max((nv.power_limit_w if nv else None) or 0, max(values, default=0), 1)
412
+ return self.hist_power, scale, " W", color
413
+ if chart == "Encoder":
414
+ return self.hist_enc, 100, "%", color
415
+ return self.hist_dec, 100, "%", color
416
+
417
+ def render_charts(self) -> None:
418
+ if self.snap is None:
419
+ return
420
+ widget = self.query_one("#charts", Static)
421
+ # Reserve the border, padding, and scrollbar even before child resize events arrive.
422
+ width = max(1, self.size.width - 6)
423
+ indices = list(self.available_charts) if self.chart_idx == -1 else [self.chart_idx]
424
+ columns = min(len(indices), 3, max(1, (width + 2) // 46))
425
+ cell_width = max(1, (width - 2 * (columns - 1)) // columns)
426
+ grid = Table.grid(padding=(0, 2), pad_edge=False, expand=True)
427
+ for _ in range(columns):
428
+ grid.add_column(width=cell_width, no_wrap=True, overflow="crop")
429
+ cells = []
430
+ for index in indices:
431
+ values, scale, unit, style = self.chart_series(index, self.snap.nv)
432
+ current = values[-1] if values else None
433
+ name = self.CHARTS[index]
434
+ if getattr(self, "_amd_chart_labels", False) and name == "SM clock":
435
+ name = "GFX clock"
436
+ cell = Text(f"{name:<12} ", style=style)
437
+ cell.append(f"{format_metric(current, '.0f', unit):>8} ", style="bold")
438
+ cell.append(spark(values, max(1, cell_width - 22), scale))
439
+ cells.append(cell)
440
+ for start in range(0, len(cells), columns):
441
+ grid.add_row(*cells[start:start + columns])
442
+ widget.update(grid)
443
+ _, power_scale, _, _ = self.chart_series(self.CHARTS.index("Power"), self.snap.nv)
444
+ _, clock_scale, _, _ = self.chart_series(self.CHARTS.index("SM clock"), self.snap.nv)
445
+ self.query_one("#chart-note", Static).update(
446
+ "Recent samples · blank = no reading" if self.snap.backend == "metal" else
447
+ f"Recent samples · blank = no reading · power 0–{power_scale:.0f} W · "
448
+ f"{'GFX' if getattr(self, '_amd_chart_labels', False) else 'SM'} clock 0–{clock_scale:.0f} MHz"
449
+ )
450
+
451
+ def on_resize(self) -> None:
452
+ if self.is_mounted and self.snap is not None:
453
+ self.query_one("#device", Static).update(self.render_device(self.snap))
454
+ self.call_after_refresh(self.render_charts)
455
+
456
+ def visible_rows(self, s: Snapshot) -> list[ProcRow]:
457
+ rows = s.procs
458
+ if not self.show_all:
459
+ rows = [r for r in rows if not r.gpu.attributed or r.gpu.dedicated >= 32 << 20 or r.gpu.gpu_pct >= 1 or r.gpu.shared >= 64 << 20
460
+ or r.gpu.paged_out >= 128 << 20 or r.llm]
461
+ rows = sorted(rows, key=self.SORTS[self.sort_idx][1])
462
+ vm = next((i for i, r in enumerate(rows) if r.kind == "wsl-vm"), None)
463
+ if vm is not None and self.SORTS[self.sort_idx][0] == "VRAM":
464
+ # Linux rows share the WSL VM row's total: keep them under it, ordered by their inferred sizes.
465
+ linux = [r for r in rows if not r.gpu.attributed and not r.host]
466
+ rest = [r for r in rows if r not in linux]
467
+ vm = rest.index(rows[vm])
468
+ rows = rest[:vm + 1] + linux + rest[vm + 1:]
469
+ return rows
470
+
471
+ def render_table(self, s: Snapshot) -> None:
472
+ t = self.query_one("#procs", DataTable)
473
+ rows = self.visible_rows(s)
474
+ keep = self.selected_key
475
+ t.clear()
476
+
477
+ def pct(v: float | None) -> Text:
478
+ if v is None:
479
+ return Text("n/a", justify="right", style="grey62")
480
+ return Text(f"{v:.0f}" if v >= 0.5 else "", justify="right", style="bold" if v >= 20 else "")
481
+
482
+ for r in rows:
483
+ e = r.gpu.engines
484
+ wddm = has_wddm_counters(s, r)
485
+ engines = has_engine_counters(s, r)
486
+ split = engines and s.backend != "metal" # Metal reports one GPU time, no per-engine split
487
+ spill = Text(gb(r.gpu.shared) if has_spill_counters(s, r) and r.gpu.shared >= 1 << 20 else "", justify="right",
488
+ style="bold red" if r.gpu.shared >= 256 << 20 and r.kind in ("LLM", "AI") and s.shared_is_spill
489
+ else "yellow")
490
+ paged = Text(gb(r.gpu.paged_out) if wddm and r.gpu.paged_out >= 1 << 20 else "", justify="right", style="grey62")
491
+ if (wddm or (s.backend == "drm" and not r.host)) and "shared" not in r.gpu.available:
492
+ spill = Text("n/a", justify="right", style="grey62")
493
+ if wddm and not r.gpu.paged_available:
494
+ paged = Text("n/a", justify="right", style="grey62")
495
+ t.add_row(
496
+ Text(r.pid_label, justify="right", style="grey62" if r.host else ""),
497
+ Text(r.name),
498
+ Text(r.kind, style=KIND_STYLE.get(r.kind, "")),
499
+ Text(" ".join(r.detail.runtimes[:3]), style="grey70"),
500
+ Text(vram_label(r.gpu), justify="right", style="bold" if r.gpu.attributed else "grey62"), spill, paged,
501
+ pct(r.gpu.gpu_pct if engines else None),
502
+ pct(e.get("3d", 0) + e.get("compute", 0) if split else None),
503
+ pct(e.get("copy", 0) if split else None),
504
+ pct(e.get("decode", 0) + e.get("encode", 0) + e.get("video", 0) if split else None),
505
+ Text(r.inside, style="magenta" if r.llm else "grey62", overflow="ellipsis", no_wrap=True),
506
+ key=r.key,
507
+ )
508
+ if keep is not None:
509
+ try:
510
+ t.move_cursor(row=t.get_row_index(keep), animate=False, scroll=False)
511
+ except Exception:
512
+ pass
513
+ t.border_subtitle = f"sort: {self.SORTS[self.sort_idx][0]} · {'all' if self.show_all else 'top'} processes"
514
+ if s.backend == "wsl" and not any(not r.host for r in s.procs):
515
+ t.border_subtitle = WSL_EMPTY
516
+
517
+ def render_inspector(self) -> None:
518
+ panel = self.query_one("#inspector", Static)
519
+ s = self.snap
520
+ r = next((x for x in s.procs if x.key == self.selected_key), None) if s else None
521
+ if r is None:
522
+ panel.update(Text("Select a process (↑/↓) to see what's inside its GPU memory. Press ? to learn what's knowable.", style="grey62"))
523
+ return
524
+ where = {"windows": "Windows PID ", "outside": "host PID "}.get(r.origin, "") if r.host else ""
525
+ panel.border_title = f"{r.name} ({where}{r.pid}) - {r.kind}"
526
+ out: list = []
527
+ e = r.gpu.engines
528
+ wddm = has_wddm_counters(s, r)
529
+ engines = " ".join(f"{k} {v:.0f}%" for k, v in e.items() if v >= 0.5) or "idle"
530
+ hdr = Text.assemble(("VRAM ", "bold"), (vram_label(r.gpu), "bold"), " spill(RAM) ",
531
+ (gb(r.gpu.shared) if has_spill_counters(s, r) else "n/a",
532
+ "bold red" if r.gpu.shared > 256 << 20 and s.shared_is_spill else ""),
533
+ " paged(est) ", gb(r.gpu.paged_out) if wddm and r.gpu.paged_available else "n/a",
534
+ " engines ", engines if has_engine_counters(s, r) else "n/a")
535
+ if r.detail.runtimes:
536
+ hdr.append(" runtime ", style="grey62")
537
+ hdr.append(" ".join(r.detail.runtimes), style="cyan")
538
+ out.append(hdr)
539
+ if not r.host and r.gpu.attributed and r.gpu.inferred_by:
540
+ out.append(Text(inferred_note(r.gpu), style="grey62"))
541
+ if not r.host and not r.gpu.attributed:
542
+ if r.gpu.inferred is not None:
543
+ out.append(Text(inferred_note(r.gpu), style="grey62"))
544
+ elif s.wsl_split:
545
+ out.append(Text(f"No per-process VRAM: {s.wsl_split}.", style="grey62"))
546
+ elif s.backend == "drm":
547
+ out.append(Text(drm_note(r.gpu), style="grey62"))
548
+ elif s.backend == "metal":
549
+ out.append(Text("GPU memory n/a: footprint can't read other users' processes without root "
550
+ "(or hasn't measured this one yet). GPU% comes from Metal's per-process GPU time.",
551
+ style="grey62"))
552
+ if s.backend == "drm" and not r.host and (r.gpu.resident_vram is not None or r.gpu.resident_gtt is not None):
553
+ def size(v): return gb(v) if v is not None else "n/a"
554
+ out.append(Text(f"Resident VRAM {size(r.gpu.resident_vram)}, GTT {size(r.gpu.resident_gtt)}. "
555
+ + DRM_RESIDENT_NOTE, style="grey62"))
556
+ if s.backend == "drm" and r.gpu.kfd:
557
+ out.append(Text("Engines n/a: this process uses ROCm/HIP (/dev/kfd), whose compute queues the kernel doesn't "
558
+ "time per process. The device's AMD gfx load includes its work.", style="grey62"))
559
+ if wddm and r.gpu.paged_out >= 64 << 20:
560
+ out.append(Text(f"{gb(r.gpu.paged_out)} of {gb(r.gpu.committed)} committed is not accounted for by local/non-local "
561
+ "usage. Estimated non-residency, not proof of eviction or disk paging.",
562
+ style="yellow"))
563
+
564
+ if r.handy:
565
+ if r.handy.selected_model:
566
+ out.append(Text(f"Handy selected (settings; not load proof): {r.handy.selected_model}", style="grey70"))
567
+ if r.handy.last_event:
568
+ out.append(Text(f"Handy last event: {r.handy.last_event} {r.handy.last_model or ''} "
569
+ f"at {r.handy.last_event_local} (runtime-reported history, not current GPU residency)",
570
+ style="green"))
571
+
572
+ v = r.llm
573
+ if v:
574
+ width = max(min(self.size.width - 8, 100), 20)
575
+ total = max(r.gpu.shown_vram or 0, sum(x.bytes for x in v.segments), 1)
576
+ out.append(stacked_bar([(x.bytes, SEG_STYLE[x.kind]) for x in v.segments], total, width))
577
+ legend = Text()
578
+ for x in v.segments:
579
+ legend.append("■ ", style=SEG_STYLE[x.kind])
580
+ legend.append(f"{x.label} {gb(x.bytes)}", style="bold" if x.exact else "")
581
+ legend.append(" ")
582
+ if v.log_source:
583
+ legend.append(f"(allocated sizes from {os.path.basename(v.log_source)})", style="green")
584
+ out.append(legend)
585
+ if v.models:
586
+ mt = Table(box=None, padding=(0, 1), show_edge=False, header_style="bold grey70")
587
+ for col in ("", "model", "arch", "params", "quant", "layers", "train ctx", "on GPU (est)", "evidence", "file"):
588
+ mt.add_column(col)
589
+ for i, g in enumerate(v.models):
590
+ f = v.files[i] if i < len(v.files) else None
591
+ mt.add_row(Text(v.kind_label(i), style="magenta"), v.label(i), g.arch, g.params_label, g.quant,
592
+ str(g.n_layers or ""), f"{g.ctx_train:,}" if g.ctx_train else "",
593
+ gb(g.gpu_weight_bytes()), Text(f.evidence if f else "", style="green"),
594
+ Text(os.path.basename(g.path), style="grey50", overflow="ellipsis"))
595
+ out.append(mt)
596
+ for f in v.other_files:
597
+ kind = f", {f.kind_label}" if f.kind_label else ""
598
+ out.append(Text(f"file: {f.store_name or f.path} ({gb(f.size)}, {f.evidence}{kind})", style="grey62"))
599
+ if v.server:
600
+ sv = v.server
601
+ st = Text.assemble(("server ", "bold"), (f"{sv.kind} {sv.url}", "cyan"))
602
+ if sv.n_ctx:
603
+ st.append(f" ctx {sv.n_ctx:,}")
604
+ if sv.slots_total is not None:
605
+ st.append(f" slots busy {sv.slots_busy}/{sv.slots_total}")
606
+ if sv.gen_tok_s:
607
+ st.append(f" gen {sv.gen_tok_s:.1f} tok/s", style="bold green")
608
+ if sv.prompt_tok_s:
609
+ st.append(f" prompt {sv.prompt_tok_s:.0f} tok/s")
610
+ for m in sv.models[:4]:
611
+ if sv.kind == "lmstudio":
612
+ st.append(f"\n loaded: {m['name']} {m.get('type') or ''} {m.get('quant') or ''} ctx {m.get('ctx') or '?'}")
613
+ out.append(st)
614
+ for n in v.notes:
615
+ out.append(Text("• " + n, style="bold red" if n.startswith("!") else "grey70"))
616
+ else:
617
+ out.append(Text(EXPLAIN.get(r.kind, ""), style="grey70"))
618
+ for m in r.detail.models:
619
+ kind = f", {m.kind_label}" if m.kind_label else ""
620
+ out.append(Text(f"file: {m.store_name or m.path} ({gb(m.size)}, {m.evidence}{kind})", style="magenta"))
621
+ if r.detail.candidates:
622
+ out.append(Text("No mapped model file was observed. Some runtimes read then close model files. "
623
+ "Installed candidates (not proven loaded):", style="grey70"))
624
+ for m in r.detail.candidates:
625
+ kind = f", {m.kind_label}" if m.kind_label else ""
626
+ out.append(Text(f" {m.store_name or m.path} ({gb(m.size)}{kind})", style="grey62"))
627
+ if r.detail.ports:
628
+ out.append(Text("listening: " + ", ".join(map(str, r.detail.ports[:8])), style="grey50"))
629
+ if r.detail.cmdline:
630
+ out.append(Text(" ".join(r.detail.cmdline)[:400], style="grey42", overflow="ellipsis"))
631
+ elif r.detail.access_denied:
632
+ out.append(Text("(protected process - run elevated for runtime/file details)", style="grey42"))
633
+ panel.update(Group(*out))
634
+
635
+ # ------------------------------------------------------------------ events/actions
636
+
637
+ def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
638
+ if event.row_key is not None and event.row_key.value is not None:
639
+ self.selected_key = str(event.row_key.value)
640
+ self.render_inspector()
641
+
642
+ def action_cycle_sort(self) -> None:
643
+ self.sort_idx = (self.sort_idx + 1) % len(self.SORTS)
644
+ if self.snap:
645
+ self.render_table(self.snap)
646
+
647
+ @property
648
+ def available_charts(self) -> list[int]:
649
+ """Chart indices this backend has data for: IOKit (macOS) has no clocks, power, temperature or codecs."""
650
+ if BACKEND == "metal":
651
+ return [i for i, name in enumerate(self.CHARTS) if name in METAL_CHARTS]
652
+ return list(range(len(self.CHARTS)))
653
+
654
+ def action_cycle_chart(self) -> None:
655
+ order = [-1, *self.available_charts]
656
+ nxt = order[(order.index(self.chart_idx) + 1) % len(order)] if self.chart_idx in order else -1
657
+ self.query_one("#chart-mode", Select).value = nxt
658
+
659
+ def on_select_changed(self, event: Select.Changed) -> None:
660
+ if event.select.id == "chart-mode" and isinstance(event.value, int):
661
+ self.chart_idx = event.value
662
+ self.render_charts()
663
+
664
+ def action_toggle_all(self) -> None:
665
+ self.show_all = not self.show_all
666
+ if self.snap:
667
+ self.render_table(self.snap)
668
+
669
+ def action_pause(self) -> None:
670
+ self.paused = not self.paused
671
+ if self.snap:
672
+ self.query_one("#device", Static).update(self.render_device(self.snap))
673
+
674
+ def action_help(self) -> None:
675
+ self.push_screen(HelpScreen())