dltop 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dltop/__init__.py +22 -0
- dltop/__main__.py +6 -0
- dltop/_version.py +8 -0
- dltop/app.py +549 -0
- dltop/cli.py +43 -0
- dltop/clipboard.py +97 -0
- dltop/export.py +131 -0
- dltop/metrics.py +92 -0
- dltop/models.py +243 -0
- dltop/sources/__init__.py +1 -0
- dltop/sources/dcgm.py +147 -0
- dltop/sources/demo.py +93 -0
- dltop/sources/nvml.py +149 -0
- dltop/sources/prometheus.py +202 -0
- dltop/sources/system.py +75 -0
- dltop/widgets/__init__.py +1 -0
- dltop/widgets/cards.py +40 -0
- dltop/widgets/plot.py +268 -0
- dltop/widgets/scroll.py +82 -0
- dltop/widgets/stats_table.py +121 -0
- dltop/widgets/toggles.py +140 -0
- dltop-0.2.0.dist-info/METADATA +172 -0
- dltop-0.2.0.dist-info/RECORD +27 -0
- dltop-0.2.0.dist-info/WHEEL +5 -0
- dltop-0.2.0.dist-info/entry_points.txt +2 -0
- dltop-0.2.0.dist-info/licenses/LICENSE +21 -0
- dltop-0.2.0.dist-info/top_level.txt +1 -0
dltop/__init__.py
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""dltop -- a top/htop-style GPU monitor tailored for CV and AI workloads.
|
|
2
|
+
|
|
3
|
+
Splits live GPU utilization into two categories:
|
|
4
|
+
|
|
5
|
+
* Compute (CV/AI): SM active, Tensor pipe, FP32/FP16/FP64 pipes -- via DCGM profiling
|
|
6
|
+
* Media engines: NVENC, NVDEC -- via NVML
|
|
7
|
+
|
|
8
|
+
The UI is built on Textual with box-drawing line charts (the nvtop/asciichart
|
|
9
|
+
look), so many metrics share one chart instead of stacking into tall bars.
|
|
10
|
+
Overlapping series are colour-interleaved per column so no line is ever hidden
|
|
11
|
+
behind another while every line keeps its true value (see ``TimeSeriesPlot``).
|
|
12
|
+
|
|
13
|
+
When DCGM profiling metrics are unavailable (e.g. consumer GeForce cards where
|
|
14
|
+
NVIDIA gates profiling fields, or DCGM not installed), we transparently fall
|
|
15
|
+
back to NVML's single overall SM% and show a status banner telling the user how
|
|
16
|
+
to enable the full split on a data-center GPU.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from dltop._version import __version__
|
|
20
|
+
from dltop.cli import main
|
|
21
|
+
|
|
22
|
+
__all__ = ["__version__", "main"]
|
dltop/__main__.py
ADDED
dltop/_version.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""Single source of the installed package version."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = version("dltop")
|
|
7
|
+
except PackageNotFoundError: # running from a checkout without an install
|
|
8
|
+
__version__ = "0.0.0+uninstalled"
|
dltop/app.py
ADDED
|
@@ -0,0 +1,549 @@
|
|
|
1
|
+
"""Main Textual app: info cards + tabbed multi-series charts + process table."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import contextlib
|
|
6
|
+
import math
|
|
7
|
+
import platform
|
|
8
|
+
import socket
|
|
9
|
+
import sys
|
|
10
|
+
import time
|
|
11
|
+
from typing import TYPE_CHECKING
|
|
12
|
+
|
|
13
|
+
from loguru import logger
|
|
14
|
+
from textual.app import App
|
|
15
|
+
from textual.binding import Binding
|
|
16
|
+
from textual.containers import Horizontal, Vertical
|
|
17
|
+
from textual.css.query import NoMatches
|
|
18
|
+
from textual.widgets import DataTable, Footer, Header, Static, TabbedContent, TabPane
|
|
19
|
+
|
|
20
|
+
from dltop._version import __version__
|
|
21
|
+
from dltop.export import CaptureMeta, GpuMeta, StatRow, host_cpu_desc
|
|
22
|
+
from dltop.metrics import MetricStore
|
|
23
|
+
from dltop.models import (
|
|
24
|
+
_PALETTE,
|
|
25
|
+
GPU_SERIES_DCGM,
|
|
26
|
+
GPU_SERIES_NVML,
|
|
27
|
+
HOST_SERIES,
|
|
28
|
+
SERIES_LABELS,
|
|
29
|
+
SERIES_UNITS,
|
|
30
|
+
SystemState,
|
|
31
|
+
_fmt_bandwidth,
|
|
32
|
+
_pct_color,
|
|
33
|
+
per_gpu,
|
|
34
|
+
split_series_name,
|
|
35
|
+
)
|
|
36
|
+
from dltop.sources.dcgm import DcgmProbe
|
|
37
|
+
from dltop.sources.demo import DemoSource
|
|
38
|
+
from dltop.sources.nvml import _collect_processes, init_nvml, pynvml, sample_nvml
|
|
39
|
+
from dltop.sources.prometheus import PromEndpoint, PromScraper, discover
|
|
40
|
+
from dltop.sources.system import init_system, psutil, sample_system
|
|
41
|
+
from dltop.widgets.cards import InfoCard
|
|
42
|
+
from dltop.widgets.plot import TimeSeriesPlot
|
|
43
|
+
from dltop.widgets.scroll import HintedScroll
|
|
44
|
+
from dltop.widgets.stats_table import StatsTable
|
|
45
|
+
from dltop.widgets.toggles import GpuToggles, SeriesToggles
|
|
46
|
+
|
|
47
|
+
if TYPE_CHECKING:
|
|
48
|
+
from psutil import Process as PsutilProcess
|
|
49
|
+
from textual.app import ComposeResult
|
|
50
|
+
|
|
51
|
+
from dltop.models import GpuState, SeriesDef
|
|
52
|
+
|
|
53
|
+
TRANSIENT_NOTES_SECONDS = 8.0
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
class DltopApp(App):
|
|
57
|
+
"""Main Textual app: info cards + tabbed multi-series charts + process table."""
|
|
58
|
+
|
|
59
|
+
CSS = """
|
|
60
|
+
Screen {
|
|
61
|
+
layout: vertical;
|
|
62
|
+
}
|
|
63
|
+
#cards-row {
|
|
64
|
+
height: auto;
|
|
65
|
+
padding: 0 1;
|
|
66
|
+
}
|
|
67
|
+
#status-banner {
|
|
68
|
+
height: auto;
|
|
69
|
+
padding: 0 1;
|
|
70
|
+
color: $warning;
|
|
71
|
+
}
|
|
72
|
+
#status-banner.hidden {
|
|
73
|
+
display: none;
|
|
74
|
+
}
|
|
75
|
+
#pause-banner {
|
|
76
|
+
height: 1;
|
|
77
|
+
padding: 0 1;
|
|
78
|
+
background: $warning;
|
|
79
|
+
color: $text;
|
|
80
|
+
text-style: bold;
|
|
81
|
+
text-align: center;
|
|
82
|
+
}
|
|
83
|
+
#pause-banner.hidden {
|
|
84
|
+
display: none;
|
|
85
|
+
}
|
|
86
|
+
TabbedContent {
|
|
87
|
+
height: 1fr;
|
|
88
|
+
min-height: 18;
|
|
89
|
+
}
|
|
90
|
+
.tab-scroll {
|
|
91
|
+
/* Subtle, thin scrollbar -- the partial next chart is the main
|
|
92
|
+
"there's more below" hint; the bar just confirms it. */
|
|
93
|
+
scrollbar-size-vertical: 1;
|
|
94
|
+
scrollbar-background: $surface;
|
|
95
|
+
scrollbar-background-hover: $surface;
|
|
96
|
+
scrollbar-background-active: $surface;
|
|
97
|
+
scrollbar-color: $panel-lighten-2;
|
|
98
|
+
scrollbar-color-hover: $accent;
|
|
99
|
+
scrollbar-color-active: $accent;
|
|
100
|
+
}
|
|
101
|
+
.chart-block {
|
|
102
|
+
height: auto;
|
|
103
|
+
}
|
|
104
|
+
.chart-block TimeSeriesPlot {
|
|
105
|
+
/* Kept short on purpose: the next chart peeks in below the fold so
|
|
106
|
+
it's obvious the tab scrolls. */
|
|
107
|
+
height: 11;
|
|
108
|
+
min-height: 8;
|
|
109
|
+
}
|
|
110
|
+
.procs {
|
|
111
|
+
height: auto;
|
|
112
|
+
border-top: solid $accent;
|
|
113
|
+
margin-top: 1;
|
|
114
|
+
}
|
|
115
|
+
.scroll-tail {
|
|
116
|
+
height: 3;
|
|
117
|
+
}
|
|
118
|
+
.paused-label {
|
|
119
|
+
color: $warning;
|
|
120
|
+
text-style: bold;
|
|
121
|
+
}
|
|
122
|
+
"""
|
|
123
|
+
|
|
124
|
+
BINDINGS = [ # noqa: RUF012
|
|
125
|
+
Binding("q", "quit", "Quit"),
|
|
126
|
+
Binding("p", "toggle_pause", "Pause"),
|
|
127
|
+
Binding("space", "toggle_pause", "Pause", show=False),
|
|
128
|
+
]
|
|
129
|
+
|
|
130
|
+
def __init__(
|
|
131
|
+
self,
|
|
132
|
+
*,
|
|
133
|
+
interval: float,
|
|
134
|
+
no_dcgm: bool,
|
|
135
|
+
demo_gpus: int | None = None,
|
|
136
|
+
window_s: float = 60.0,
|
|
137
|
+
no_discover: bool = False,
|
|
138
|
+
) -> None:
|
|
139
|
+
"""Configure sampling ``interval`` (seconds) and whether to skip DCGM.
|
|
140
|
+
|
|
141
|
+
``demo_gpus`` selects the synthetic :class:`DemoSource` in place of real NVML/DCGM
|
|
142
|
+
hardware (``None`` disables demo mode). ``window_s`` is the stats window for the
|
|
143
|
+
Table tab. ``no_discover`` disables Prometheus ``/metrics`` auto-discovery.
|
|
144
|
+
"""
|
|
145
|
+
super().__init__()
|
|
146
|
+
self.interval = interval
|
|
147
|
+
self.no_dcgm = no_dcgm
|
|
148
|
+
self.demo = DemoSource(demo_gpus) if demo_gpus else None
|
|
149
|
+
self.window_s = window_s
|
|
150
|
+
self.no_discover = no_discover
|
|
151
|
+
self._t0 = time.monotonic()
|
|
152
|
+
self.store = MetricStore()
|
|
153
|
+
self.gpus: list[GpuState] = []
|
|
154
|
+
self.sys_state: SystemState = SystemState()
|
|
155
|
+
self.dcgm: DcgmProbe | None = None
|
|
156
|
+
self.have_profiling = False
|
|
157
|
+
self.dcgm_note = ""
|
|
158
|
+
self.prom_endpoints: list[PromEndpoint] = []
|
|
159
|
+
self.prom: PromScraper | None = None
|
|
160
|
+
self._prom_note = ""
|
|
161
|
+
self.paused = False
|
|
162
|
+
self._nv_driver = ""
|
|
163
|
+
self._cuda_ver = ""
|
|
164
|
+
self._proc_cache: dict[int, PsutilProcess] = {}
|
|
165
|
+
|
|
166
|
+
# -- bring-up --------------------------------------------------------------------
|
|
167
|
+
|
|
168
|
+
def _prepare_sources(self) -> None:
|
|
169
|
+
if self.demo is not None:
|
|
170
|
+
self.gpus = self.demo.init_gpus()
|
|
171
|
+
self.sys_state = SystemState()
|
|
172
|
+
self.have_profiling = True # demo fabricates the full DCGM series
|
|
173
|
+
self._nv_driver, self._cuda_ver = "demo", "demo"
|
|
174
|
+
else:
|
|
175
|
+
self.gpus = init_nvml()
|
|
176
|
+
if not self.gpus:
|
|
177
|
+
sys.exit("No NVIDIA GPUs detected by NVML")
|
|
178
|
+
self.sys_state = init_system()
|
|
179
|
+
|
|
180
|
+
with contextlib.suppress(pynvml.NVMLError):
|
|
181
|
+
raw = pynvml.nvmlSystemGetDriverVersion()
|
|
182
|
+
self._nv_driver = raw.decode() if isinstance(raw, bytes) else raw
|
|
183
|
+
with contextlib.suppress(pynvml.NVMLError):
|
|
184
|
+
v = pynvml.nvmlSystemGetCudaDriverVersion()
|
|
185
|
+
self._cuda_ver = f"{v // 1000}.{(v % 1000) // 10}"
|
|
186
|
+
|
|
187
|
+
if self.no_dcgm:
|
|
188
|
+
self.dcgm_note = "DCGM disabled by --no-dcgm. Showing NVML overall SM% (lumped CUDA+Tensor+RT)."
|
|
189
|
+
else:
|
|
190
|
+
ok_cli, err_cli = DcgmProbe.cli_available()
|
|
191
|
+
if not ok_cli:
|
|
192
|
+
self.dcgm_note = f"DCGM unavailable: {err_cli}"
|
|
193
|
+
else:
|
|
194
|
+
ok_prof, err_prof = DcgmProbe.profiling_supported()
|
|
195
|
+
if ok_prof:
|
|
196
|
+
self.dcgm = DcgmProbe(len(self.gpus), interval_ms=int(self.interval * 1000))
|
|
197
|
+
self.dcgm.start()
|
|
198
|
+
self.have_profiling = True
|
|
199
|
+
else:
|
|
200
|
+
self.dcgm_note = f"DCGM present but profiling fields not available on this GPU. {err_prof}"
|
|
201
|
+
|
|
202
|
+
if not self.no_discover:
|
|
203
|
+
self.prom_endpoints = discover()
|
|
204
|
+
if self.prom_endpoints:
|
|
205
|
+
self.prom = PromScraper(self.prom_endpoints, self.store, interval_s=max(self.interval, 1.0))
|
|
206
|
+
self.prom.start()
|
|
207
|
+
found = ", ".join(f":{ep.port} ({ep.name}, {len(ep.metrics)} metrics)" for ep in self.prom_endpoints)
|
|
208
|
+
self._prom_note = f"Prometheus: scraping {found}"
|
|
209
|
+
else:
|
|
210
|
+
self._prom_note = ""
|
|
211
|
+
|
|
212
|
+
def _gpu_series_table(self) -> dict[str, list[SeriesDef]]:
|
|
213
|
+
"""Return the per-GPU series table: DCGM's richer split, or NVML's lumped SM%."""
|
|
214
|
+
return GPU_SERIES_DCGM if self.have_profiling else GPU_SERIES_NVML
|
|
215
|
+
|
|
216
|
+
def _chart_block(self, plot_id: str, title: str, series: list[SeriesDef], *, classes: str = "") -> Vertical:
|
|
217
|
+
"""Wrap one chart + its series toggles in a fixed-height ``Vertical`` block."""
|
|
218
|
+
return Vertical(
|
|
219
|
+
TimeSeriesPlot(self.store, series, title, plot_id=plot_id),
|
|
220
|
+
SeriesToggles(plot_id, series),
|
|
221
|
+
classes=f"chart-block {classes}".strip(),
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
def _compose_tab(self, tab: str) -> ComposeResult:
|
|
225
|
+
"""Yield the host chart followed by one chart per GPU, all for domain ``tab``."""
|
|
226
|
+
host_title = "Host — CPU · RAM · Disk · Network" if tab == "all" else f"Host — {tab}"
|
|
227
|
+
yield self._chart_block(f"{tab}-host-plot", host_title, HOST_SERIES[tab])
|
|
228
|
+
for g in self.gpus:
|
|
229
|
+
yield self._chart_block(
|
|
230
|
+
f"{tab}-gpu{g.index}-plot",
|
|
231
|
+
f"GPU {g.index} · {g.name}",
|
|
232
|
+
per_gpu(self._gpu_series_table()[tab], g.index),
|
|
233
|
+
classes=f"gpu-chart-{g.index}",
|
|
234
|
+
)
|
|
235
|
+
if tab == "all" and self.prom_endpoints:
|
|
236
|
+
series = self._prom_series_defs()
|
|
237
|
+
plot = TimeSeriesPlot(
|
|
238
|
+
self.store,
|
|
239
|
+
series,
|
|
240
|
+
"Prometheus — % of peak ◍",
|
|
241
|
+
plot_id="all-prom-plot",
|
|
242
|
+
scale_to_peak=True,
|
|
243
|
+
)
|
|
244
|
+
yield Vertical(plot, SeriesToggles("all-prom-plot", series), classes="chart-block")
|
|
245
|
+
# Processes ride at the very bottom of the same scroll, so the whole tab
|
|
246
|
+
# is one continuous top-to-bottom read rather than two separate panes.
|
|
247
|
+
yield DataTable(zebra_stripes=True, classes="procs")
|
|
248
|
+
# A little dead space after the table so scrolling to the end visibly
|
|
249
|
+
# bottoms out -- it's obvious there's nothing more below.
|
|
250
|
+
yield Static("", classes="scroll-tail")
|
|
251
|
+
|
|
252
|
+
def _prom_series_defs(self) -> list[SeriesDef]:
|
|
253
|
+
"""One series per discovered metric, sorted by name and on by default."""
|
|
254
|
+
palette = list(_PALETTE)
|
|
255
|
+
pairs = sorted(
|
|
256
|
+
((ep.port, metric) for ep in self.prom_endpoints for metric in ep.metrics),
|
|
257
|
+
key=lambda pm: (pm[1], pm[0]),
|
|
258
|
+
)
|
|
259
|
+
defs: list[SeriesDef] = []
|
|
260
|
+
for n, (port, metric) in enumerate(pairs):
|
|
261
|
+
label = f"◍ {metric[:28]} :{port}"
|
|
262
|
+
defs.append((f"prom:{port}:{metric}", palette[n % len(palette)], label, True))
|
|
263
|
+
return defs
|
|
264
|
+
|
|
265
|
+
# -- layout ----------------------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
def compose(self) -> ComposeResult: # noqa: D102
|
|
268
|
+
self._prepare_sources()
|
|
269
|
+
yield Header(show_clock=True, name=self._header_name())
|
|
270
|
+
|
|
271
|
+
yield Static("", id="status-banner", classes="hidden")
|
|
272
|
+
yield Static("⏸ PAUSED — press p to resume", id="pause-banner", classes="hidden")
|
|
273
|
+
|
|
274
|
+
with Horizontal(id="cards-row"):
|
|
275
|
+
for g in self.gpus:
|
|
276
|
+
yield InfoCard(f" GPU {g.index} · {g.name} ", card_id=f"gpu-card-{g.index}")
|
|
277
|
+
yield InfoCard(" Memory ", card_id=f"mem-card-{g.index}")
|
|
278
|
+
yield InfoCard(" Power / Thermal ", card_id=f"power-card-{g.index}")
|
|
279
|
+
yield InfoCard(" PCIe ", card_id=f"pcie-card-{g.index}")
|
|
280
|
+
yield InfoCard(" System ", card_id="sys-card")
|
|
281
|
+
|
|
282
|
+
if len(self.gpus) > 1:
|
|
283
|
+
yield GpuToggles(self.gpus)
|
|
284
|
+
with TabbedContent(id="tabs"):
|
|
285
|
+
for tab, title in (("all", "All"), ("compute", "Compute"), ("memory", "Memory"), ("system", "System")):
|
|
286
|
+
with TabPane(title, id=f"tab-{tab}"):
|
|
287
|
+
yield HintedScroll(*self._compose_tab(tab))
|
|
288
|
+
with TabPane("Table", id="tab-table"):
|
|
289
|
+
yield StatsTable(self.window_s, self._stat_rows, self._capture_meta)
|
|
290
|
+
|
|
291
|
+
yield Footer()
|
|
292
|
+
|
|
293
|
+
def _header_name(self) -> str:
|
|
294
|
+
mode = "DEMO" if self.demo is not None else ("DCGM" if self.have_profiling else "NVML")
|
|
295
|
+
return f"dltop · driver {self._nv_driver or '?'} · CUDA {self._cuda_ver or '?'} · mode: {mode}"
|
|
296
|
+
|
|
297
|
+
# -- on_mount: prime things that need widgets to exist ----------------------------
|
|
298
|
+
|
|
299
|
+
def on_mount(self) -> None: # noqa: D102
|
|
300
|
+
self.title = "dltop"
|
|
301
|
+
self.sub_title = self._header_name()
|
|
302
|
+
|
|
303
|
+
for procs in self.query(".procs").results(DataTable):
|
|
304
|
+
procs.add_columns("GPU", "PID", "USER", "TYPE", "GPU-MEM", "%CPU", "HOST-MEM", "COMMAND")
|
|
305
|
+
procs.cursor_type = "row"
|
|
306
|
+
|
|
307
|
+
banner_text = "\n".join(note for note in (self.dcgm_note, self._prom_note) if note)
|
|
308
|
+
if banner_text:
|
|
309
|
+
banner = self.query_one("#status-banner", Static)
|
|
310
|
+
banner.update(banner_text)
|
|
311
|
+
banner.remove_class("hidden")
|
|
312
|
+
self.set_timer(TRANSIENT_NOTES_SECONDS, lambda: banner.add_class("hidden"))
|
|
313
|
+
|
|
314
|
+
self.set_interval(self.interval, self._tick)
|
|
315
|
+
# One immediate sample so graphs aren't empty on first paint. Deferred via
|
|
316
|
+
# call_after_refresh (not a direct call) because widgets composed inside
|
|
317
|
+
# TabbedContent/Horizontal may not be mounted yet when on_mount fires on newer
|
|
318
|
+
# Textual; a synchronous _tick() then races query_one() -> NoMatches and crashes.
|
|
319
|
+
self.call_after_refresh(self._tick)
|
|
320
|
+
|
|
321
|
+
# -- sampling & redraw -----------------------------------------------------------
|
|
322
|
+
|
|
323
|
+
def _tick(self) -> None:
|
|
324
|
+
if self.paused:
|
|
325
|
+
return
|
|
326
|
+
if self.demo is not None:
|
|
327
|
+
self.demo.sample(self.gpus, self.sys_state, time.monotonic() - self._t0)
|
|
328
|
+
else:
|
|
329
|
+
for g in self.gpus:
|
|
330
|
+
sample_nvml(g)
|
|
331
|
+
if self.dcgm is not None:
|
|
332
|
+
g.dcgm = self.dcgm.snapshot(g.index)
|
|
333
|
+
sample_system(self.sys_state)
|
|
334
|
+
self._push_series()
|
|
335
|
+
# A `set_interval` timer callback can still be in flight when the app starts
|
|
336
|
+
# tearing down (e.g. a test's `run_test()` context exiting) -- widgets may
|
|
337
|
+
# already be unmounted by the time this runs. That's not a real bug, so
|
|
338
|
+
# swallow it the same way TimeSeriesPlot.render() never lets a draw glitch
|
|
339
|
+
# kill the whole monitor.
|
|
340
|
+
try:
|
|
341
|
+
self._refresh_cards()
|
|
342
|
+
self._refresh_procs()
|
|
343
|
+
self.query_one(StatsTable).refresh_stats()
|
|
344
|
+
except NoMatches:
|
|
345
|
+
# Benign at teardown (widgets already gone); during normal operation
|
|
346
|
+
# it would signal a real id/query mismatch, so leave a debug trace
|
|
347
|
+
# rather than freezing the panels completely silently.
|
|
348
|
+
logger.debug("_tick refresh skipped: queried a widget that is not mounted")
|
|
349
|
+
|
|
350
|
+
def _push_series(self) -> None:
|
|
351
|
+
"""Record one sample per series: plain names for host, ``name@{i}`` per GPU.
|
|
352
|
+
|
|
353
|
+
Values are raw (unclamped) -- the plot clamps its own y-axis visually, but
|
|
354
|
+
the Table tab (Task 6) needs the true numbers, e.g. actual MB/s not a
|
|
355
|
+
percent-axis-friendly cap.
|
|
356
|
+
"""
|
|
357
|
+
samples: dict[str, float] = {
|
|
358
|
+
"cpu": self.sys_state.cpu_pct,
|
|
359
|
+
"ram": self.sys_state.ram_pct,
|
|
360
|
+
"disk_r": self.sys_state.disk_read_mbs,
|
|
361
|
+
"disk_w": self.sys_state.disk_write_mbs,
|
|
362
|
+
"net_rx": self.sys_state.net_rx_mbs,
|
|
363
|
+
"net_tx": self.sys_state.net_tx_mbs,
|
|
364
|
+
}
|
|
365
|
+
for g in self.gpus:
|
|
366
|
+
i = g.index
|
|
367
|
+
samples[f"nvenc@{i}"] = g.nvenc_pct
|
|
368
|
+
samples[f"nvdec@{i}"] = g.nvdec_pct
|
|
369
|
+
samples[f"vram@{i}"] = g.mem_pct
|
|
370
|
+
samples[f"mbw@{i}"] = g.mbw_pct
|
|
371
|
+
samples[f"power@{i}"] = (g.power_w / g.power_limit_w * 100.0) if g.power_limit_w else 0.0
|
|
372
|
+
samples[f"pcie_tx@{i}"] = (g.pcie_tx_mbs / g.pcie_max_mbs * 100.0) if g.pcie_max_mbs else 0.0
|
|
373
|
+
samples[f"pcie_rx@{i}"] = (g.pcie_rx_mbs / g.pcie_max_mbs * 100.0) if g.pcie_max_mbs else 0.0
|
|
374
|
+
if self.have_profiling:
|
|
375
|
+
samples[f"sm@{i}"] = g.dcgm.get("sm_active", 0.0)
|
|
376
|
+
samples[f"tensor@{i}"] = g.dcgm.get("tensor_active", 0.0)
|
|
377
|
+
samples[f"fp32@{i}"] = g.dcgm.get("fp32_active", 0.0)
|
|
378
|
+
samples[f"fp16@{i}"] = g.dcgm.get("fp16_active", 0.0)
|
|
379
|
+
samples[f"fp64@{i}"] = g.dcgm.get("fp64_active", 0.0)
|
|
380
|
+
else:
|
|
381
|
+
samples[f"sm@{i}"] = g.sm_overall_pct
|
|
382
|
+
self.store.record_many(samples, ts=time.time())
|
|
383
|
+
for plot in self.query(TimeSeriesPlot):
|
|
384
|
+
plot.replot()
|
|
385
|
+
|
|
386
|
+
def _refresh_cards(self) -> None:
|
|
387
|
+
for g in self.gpus:
|
|
388
|
+
self.query_one(f"#gpu-card-{g.index}", InfoCard).update_rows(
|
|
389
|
+
[
|
|
390
|
+
("Temp", f"[{_pct_color((g.temp_c - 30) * 2)}]{g.temp_c:.0f}°C[/]"),
|
|
391
|
+
("Fan", "—" if math.isnan(g.fan_pct) else f"{g.fan_pct:.0f}%"),
|
|
392
|
+
("SM clk", f"{g.sm_clock_mhz:.0f} MHz"),
|
|
393
|
+
("Mem clk", f"{g.mem_clock_mhz:.0f} MHz"),
|
|
394
|
+
],
|
|
395
|
+
)
|
|
396
|
+
self.query_one(f"#mem-card-{g.index}", InfoCard).update_rows(
|
|
397
|
+
[
|
|
398
|
+
(
|
|
399
|
+
"VRAM",
|
|
400
|
+
f"[{_pct_color(g.mem_pct)}]{g.mem_used_mb / 1024:.2f}"
|
|
401
|
+
f" / {g.mem_total_mb / 1024:.2f} GiB[/] ({g.mem_pct:.0f}%)",
|
|
402
|
+
),
|
|
403
|
+
("Bandwidth", f"[{_pct_color(g.mbw_pct)}]{g.mbw_pct:.0f}%[/] util"),
|
|
404
|
+
],
|
|
405
|
+
)
|
|
406
|
+
power_pct = (g.power_w / g.power_limit_w * 100.0) if g.power_limit_w else 0.0
|
|
407
|
+
self.query_one(f"#power-card-{g.index}", InfoCard).update_rows(
|
|
408
|
+
[
|
|
409
|
+
(
|
|
410
|
+
"Power",
|
|
411
|
+
f"[{_pct_color(power_pct)}]{g.power_w:.0f}"
|
|
412
|
+
f" / {g.power_limit_w:.0f} W[/] ({power_pct:.0f}%)",
|
|
413
|
+
),
|
|
414
|
+
("NVENC", f"[yellow]{g.nvenc_pct:.0f}%[/]"),
|
|
415
|
+
("NVDEC", f"[yellow]{g.nvdec_pct:.0f}%[/]"),
|
|
416
|
+
],
|
|
417
|
+
)
|
|
418
|
+
link_tag = f"Gen{g.pcie_curr_gen}x{g.pcie_curr_width}" if g.pcie_curr_gen else "?"
|
|
419
|
+
tx_pct = (g.pcie_tx_mbs / g.pcie_max_mbs * 100.0) if g.pcie_max_mbs else 0.0
|
|
420
|
+
rx_pct = (g.pcie_rx_mbs / g.pcie_max_mbs * 100.0) if g.pcie_max_mbs else 0.0
|
|
421
|
+
self.query_one(f"#pcie-card-{g.index}", InfoCard).update_rows(
|
|
422
|
+
[
|
|
423
|
+
("Link", f"[bright_white]{link_tag}[/]"),
|
|
424
|
+
("TX", f"[{_pct_color(tx_pct)}]{_fmt_bandwidth(g.pcie_tx_mbs)}[/] ({tx_pct:.1f}%)"),
|
|
425
|
+
("RX", f"[{_pct_color(rx_pct)}]{_fmt_bandwidth(g.pcie_rx_mbs)}[/] ({rx_pct:.1f}%)"),
|
|
426
|
+
],
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
sys_rows: list[tuple[str, str]] = []
|
|
430
|
+
if psutil is None:
|
|
431
|
+
sys_rows.append(("psutil", "not installed"))
|
|
432
|
+
else:
|
|
433
|
+
sys_rows.extend(
|
|
434
|
+
[
|
|
435
|
+
(
|
|
436
|
+
"CPU",
|
|
437
|
+
f"[{_pct_color(self.sys_state.cpu_pct)}]{self.sys_state.cpu_pct:.0f}%[/]"
|
|
438
|
+
f" ({self.sys_state.cpu_count_physical}C/{self.sys_state.cpu_count_logical}T)",
|
|
439
|
+
),
|
|
440
|
+
(
|
|
441
|
+
"Load",
|
|
442
|
+
f"{self.sys_state.load_1:.2f} {self.sys_state.load_5:.2f} {self.sys_state.load_15:.2f}",
|
|
443
|
+
),
|
|
444
|
+
(
|
|
445
|
+
"RAM",
|
|
446
|
+
f"[{_pct_color(self.sys_state.ram_pct)}]{self.sys_state.ram_used_gib:.1f}"
|
|
447
|
+
f" / {self.sys_state.ram_total_gib:.1f} GiB[/]",
|
|
448
|
+
),
|
|
449
|
+
(
|
|
450
|
+
"IO",
|
|
451
|
+
f"d {self.sys_state.disk_read_mbs:.1f}/{self.sys_state.disk_write_mbs:.1f}"
|
|
452
|
+
f" n {self.sys_state.net_rx_mbs:.1f}/{self.sys_state.net_tx_mbs:.1f} MB/s",
|
|
453
|
+
),
|
|
454
|
+
],
|
|
455
|
+
)
|
|
456
|
+
self.query_one("#sys-card", InfoCard).update_rows(sys_rows)
|
|
457
|
+
|
|
458
|
+
def _refresh_procs(self) -> None:
|
|
459
|
+
# Enumerate once; the process table is composed once per chart tab so it
|
|
460
|
+
# sits at the bottom of each tab's scroll -- repaint every copy.
|
|
461
|
+
procs = self.demo.processes() if self.demo is not None else _collect_processes(self.gpus, self._proc_cache)
|
|
462
|
+
if procs:
|
|
463
|
+
rows = [
|
|
464
|
+
(
|
|
465
|
+
str(p["gpu"]),
|
|
466
|
+
str(p["pid"]),
|
|
467
|
+
p["user"],
|
|
468
|
+
p["type"],
|
|
469
|
+
f"{p['mem_mb']:.0f} MiB",
|
|
470
|
+
"—" if math.isnan(p["cpu_pct"]) else f"{p['cpu_pct']:.0f}",
|
|
471
|
+
"—" if math.isnan(p["rss_mb"]) else f"{p['rss_mb']:.0f}M",
|
|
472
|
+
p["cmd"][:120],
|
|
473
|
+
)
|
|
474
|
+
for p in procs
|
|
475
|
+
]
|
|
476
|
+
else:
|
|
477
|
+
rows = [("—", "—", "—", "—", "—", "—", "—", "(no GPU processes)")]
|
|
478
|
+
for table in self.query(".procs").results(DataTable):
|
|
479
|
+
table.clear()
|
|
480
|
+
for row in rows:
|
|
481
|
+
table.add_row(*row)
|
|
482
|
+
|
|
483
|
+
# -- Table tab (Task 6) -----------------------------------------------------------
|
|
484
|
+
|
|
485
|
+
def _stat_rows(self) -> list[StatRow]:
|
|
486
|
+
"""Every recorded series as a StatRow: host first, then GPUs, then Prometheus."""
|
|
487
|
+
rows: list[StatRow] = []
|
|
488
|
+
for name in sorted(self.store.names(), key=self._row_sort_key):
|
|
489
|
+
stats = self.store.stats(name, self.window_s)
|
|
490
|
+
if stats is None:
|
|
491
|
+
continue
|
|
492
|
+
if name.startswith("prom:"):
|
|
493
|
+
_, port, metric = name.split(":", 2)
|
|
494
|
+
rows.append(StatRow(metric, f":{port}", "", stats))
|
|
495
|
+
continue
|
|
496
|
+
base, gpu_idx = split_series_name(name)
|
|
497
|
+
label = SERIES_LABELS.get(base, base)
|
|
498
|
+
source = "host" if gpu_idx is None else f"GPU {gpu_idx}"
|
|
499
|
+
rows.append(StatRow(label, source, SERIES_UNITS.get(base, ""), stats))
|
|
500
|
+
return rows
|
|
501
|
+
|
|
502
|
+
@staticmethod
|
|
503
|
+
def _row_sort_key(name: str) -> tuple[int, str]:
|
|
504
|
+
if name.startswith("prom:"):
|
|
505
|
+
return (2, name)
|
|
506
|
+
return (1, name) if "@" in name else (0, name)
|
|
507
|
+
|
|
508
|
+
def _capture_meta(self) -> CaptureMeta:
|
|
509
|
+
"""Snapshot of measurement context for the Copy-metadata button."""
|
|
510
|
+
n = self.store.stats("cpu", self.window_s)
|
|
511
|
+
return CaptureMeta(
|
|
512
|
+
captured_at=time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
513
|
+
version=__version__,
|
|
514
|
+
window_s=self.window_s,
|
|
515
|
+
n_samples=n.n_samples if n else 0,
|
|
516
|
+
interval_s=self.interval,
|
|
517
|
+
hostname=socket.gethostname(),
|
|
518
|
+
os_desc=f"{platform.system()} {platform.release()}",
|
|
519
|
+
cpu_desc=(f"{host_cpu_desc()} · {self.sys_state.cpu_count_physical}C/{self.sys_state.cpu_count_logical}T"),
|
|
520
|
+
ram_gib=self.sys_state.ram_total_gib,
|
|
521
|
+
gpus=[GpuMeta(g.index, g.name, g.mem_total_mb / 1024.0) for g in self.gpus],
|
|
522
|
+
driver=self._nv_driver or "?",
|
|
523
|
+
cuda=self._cuda_ver or "?",
|
|
524
|
+
prom_endpoints=[f":{ep.port} ({ep.name}, {len(ep.metrics)} metrics)" for ep in self.prom_endpoints],
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
# -- bindings --------------------------------------------------------------------
|
|
528
|
+
|
|
529
|
+
def action_toggle_pause(self) -> None:
|
|
530
|
+
"""Freeze sampling; readouts and graphs hold their last values."""
|
|
531
|
+
self.paused = not self.paused
|
|
532
|
+
banner = self.query_one("#pause-banner", Static)
|
|
533
|
+
if self.paused:
|
|
534
|
+
banner.remove_class("hidden")
|
|
535
|
+
self.sub_title = self._header_name() + " ⏸ PAUSED"
|
|
536
|
+
else:
|
|
537
|
+
banner.add_class("hidden")
|
|
538
|
+
self.sub_title = self._header_name()
|
|
539
|
+
|
|
540
|
+
# -- shutdown --------------------------------------------------------------------
|
|
541
|
+
|
|
542
|
+
def on_unmount(self) -> None: # noqa: D102
|
|
543
|
+
if self.dcgm is not None:
|
|
544
|
+
self.dcgm.stop()
|
|
545
|
+
if self.prom is not None:
|
|
546
|
+
self.prom.stop()
|
|
547
|
+
if self.demo is None:
|
|
548
|
+
with contextlib.suppress(pynvml.NVMLError):
|
|
549
|
+
pynvml.nvmlShutdown()
|
dltop/cli.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""CLI entry point."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
from loguru import logger
|
|
9
|
+
|
|
10
|
+
from dltop._version import __version__
|
|
11
|
+
from dltop.app import DltopApp
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main() -> None:
|
|
15
|
+
"""CLI entry point."""
|
|
16
|
+
ap = argparse.ArgumentParser(description="dltop -- a top/htop-style GPU monitor for CV and AI workloads.")
|
|
17
|
+
ap.add_argument("-i", "--interval", type=float, default=0.5, help="sampling interval, seconds")
|
|
18
|
+
ap.add_argument("--no-dcgm", action="store_true", help="skip DCGM probe entirely")
|
|
19
|
+
ap.add_argument("--window", type=float, default=60.0, help="stats window for the Table tab, seconds")
|
|
20
|
+
ap.add_argument("--no-discover", action="store_true", help="disable Prometheus /metrics auto-discovery")
|
|
21
|
+
ap.add_argument(
|
|
22
|
+
"--demo",
|
|
23
|
+
type=int,
|
|
24
|
+
nargs="?",
|
|
25
|
+
const=2,
|
|
26
|
+
default=None,
|
|
27
|
+
metavar="N",
|
|
28
|
+
help="synthetic N-GPU telemetry (default 2); no NVIDIA hardware needed",
|
|
29
|
+
)
|
|
30
|
+
ap.add_argument("--version", action="version", version=f"dltop {__version__}")
|
|
31
|
+
args = ap.parse_args()
|
|
32
|
+
|
|
33
|
+
logger.remove()
|
|
34
|
+
logger.add(sys.stderr, level="WARNING")
|
|
35
|
+
|
|
36
|
+
app = DltopApp(
|
|
37
|
+
interval=args.interval,
|
|
38
|
+
no_dcgm=args.no_dcgm,
|
|
39
|
+
demo_gpus=args.demo,
|
|
40
|
+
window_s=args.window,
|
|
41
|
+
no_discover=args.no_discover,
|
|
42
|
+
)
|
|
43
|
+
app.run()
|