llmperfscope 0.1.0__py3-none-any.whl

llmperfscope/gpu.py

@@ -0,0 +1,277 @@
+"""GPU telemetry. NVML today; the interface is small so other vendors can plug in.
+
+Everything here works without admin rights. Hardware counters (cache hit rate, per-kernel DRAM traffic) need
+RmProfilingAdminOnly=0 or root and go through `nsys --gpu-metrics-devices` instead; see capabilities.py.
+"""
+
+from __future__ import annotations
+
+import ctypes as C
+import os
+
+# Columns of one device in the fast sample.
+G_MEM, G_POWER, G_ENERGY, G_SM, G_MEMCLK, G_UTIL, G_UTILMEM, G_THROTTLE, G_TEMP = range(9)
+
+THROTTLE_BITS = {
+    0x1: "idle",
+    0x2: "app_clocks",
+    0x4: "sw_power_cap",
+    0x8: "hw_slowdown",
+    0x10: "sync_boost",
+    0x20: "sw_thermal",
+    0x40: "hw_thermal",
+    0x80: "hw_power_brake",
+    0x100: "display_clocks",
+}
+# Reasons that mean the GPU was held below what the work wanted.
+THROTTLE_BAD = 0x4 | 0x8 | 0x20 | 0x40 | 0x80
+# Reasons that are real even on an idle GPU. The software power cap bit is not: idle GPUs report it with nothing to cap.
+THROTTLE_HOT = 0x8 | 0x20 | 0x40 | 0x80
+
+
+def throttle_names(bits, include_idle=False):
+    return [name for bit, name in THROTTLE_BITS.items() if (bits & bit) and (include_idle or bit != 0x1)]
+
+
+class GpuBackend:
+    """No GPU telemetry: runs still get host metrics and the phase timeline."""
+
+    name = "none"
+
+    def __init__(self, reason="no backend"):
+        self.reason = reason
+        self.devices = []
+
+    def fast(self):
+        """Cheap gauges, per device: mem MiB, W, mJ (None here, see energy_mj), SM MHz, mem MHz, util %, mem util %, throttle bits, C."""
+        return []
+
+    def energy_mj(self, index):
+        """The device's cumulative energy counter. Can block far longer than the gauges, so it has its own thread."""
+        return None
+
+    def slow(self):
+        """Costlier reads, per device: {"procs": {pid: MiB}, "nvlink": (tx KiB, rx KiB) or None}."""
+        return []
+
+    def util_samples(self):
+        """Per device, new utilization samples as (wall ns, %). Empty when the backend keeps no history."""
+        return [[] for _ in self.devices]
+
+    def pcie_kbps(self, index, direction):
+        return None
+
+    def driver_version(self):
+        return None
+
+    def close(self):
+        pass
+
+
+class _Memory(C.Structure):
+    _fields_ = [("total", C.c_ulonglong), ("free", C.c_ulonglong), ("used", C.c_ulonglong)]
+
+
+class _Utilization(C.Structure):
+    _fields_ = [("gpu", C.c_uint), ("memory", C.c_uint)]
+
+
+class _ProcessInfo(C.Structure):
+    _fields_ = [("pid", C.c_uint), ("used", C.c_ulonglong), ("gi", C.c_uint), ("ci", C.c_uint)]
+
+
+class _FieldUnion(C.Union):
+    _fields_ = [("d", C.c_double), ("ui", C.c_uint), ("ul", C.c_ulong), ("ull", C.c_ulonglong), ("sll", C.c_longlong)]
+
+
+class _Sample(C.Structure):
+    _fields_ = [("timeStamp", C.c_ulonglong), ("value", C.c_ulonglong)]
+
+
+class _FieldValue(C.Structure):
+    _fields_ = [
+        ("fieldId", C.c_uint),
+        ("scopeId", C.c_uint),
+        ("timestamp", C.c_longlong),
+        ("latencyUsec", C.c_longlong),
+        ("valueType", C.c_int),
+        ("nvmlReturn", C.c_int),
+        ("value", _FieldUnion),
+    ]
+
+
+_FI_NVLINK_TX = 138  # NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_TX, KiB
+_FI_NVLINK_RX = 139  # NVML_FI_DEV_NVLINK_THROUGHPUT_DATA_RX, KiB
+_ALL_LINKS = 0xFFFFFFFF
+_NOT_AVAILABLE = 0xFFFFFFFFFFFFFFFF
+_MAX_PROCS = 256
+_MIB = 1024 * 1024
+
+
+class NvmlBackend(GpuBackend):
+    name = "nvml"
+
+    def __init__(self, library="libnvidia-ml.so.1"):
+        super().__init__(reason="")
+        self.lib = C.CDLL(library)
+        rc = self.lib.nvmlInit_v2()
+        if rc != 0:
+            raise OSError(f"nvmlInit_v2 returned {rc}")
+        self.handles = []
+        count = C.c_uint()
+        self.lib.nvmlDeviceGetCount_v2(C.byref(count))
+        for i in range(count.value):
+            handle = C.c_void_p()
+            if self.lib.nvmlDeviceGetHandleByIndex_v2(i, C.byref(handle)) != 0:
+                continue
+            self.handles.append(handle)
+            self.devices.append(self._describe(i, handle))
+
+        self._throttle = None
+        for fn in ("nvmlDeviceGetCurrentClocksEventReasons", "nvmlDeviceGetCurrentClocksThrottleReasons"):
+            if hasattr(self.lib, fn):
+                self._throttle = getattr(self.lib, fn)
+                break
+        self._procs = None
+        for fn in ("nvmlDeviceGetComputeRunningProcesses_v3", "nvmlDeviceGetComputeRunningProcesses_v2"):
+            if hasattr(self.lib, fn):
+                self._procs = getattr(self.lib, fn)
+                break
+        self._proc_buffer = (_ProcessInfo * _MAX_PROCS)()
+        self._mem = _Memory()
+        self._util = _Utilization()
+        self._uint = C.c_uint()
+        self._ull = C.c_ulonglong()
+        self._nvlink_ok = [self._probe_nvlink(h) for h in self.handles]
+        for dev, ok in zip(self.devices, self._nvlink_ok):
+            dev["nvlink"] = ok
+        self._samples = (_Sample * 512)()
+        self._util_seen = [0 for _ in self.handles]
+
+    def driver_version(self):
+        text = C.create_string_buffer(96)
+        if self.lib.nvmlSystemGetDriverVersion(text, 96) == 0:
+            return text.value.decode()
+        return None
+
+    def _describe(self, index, handle):
+        name = C.create_string_buffer(96)
+        uuid = C.create_string_buffer(96)
+        self.lib.nvmlDeviceGetName(handle, name, 96)
+        self.lib.nvmlDeviceGetUUID(handle, uuid, 96)
+        mem = _Memory()
+        self.lib.nvmlDeviceGetMemoryInfo(handle, C.byref(mem))
+        limit = C.c_uint()
+        power_limit = self.lib.nvmlDeviceGetEnforcedPowerLimit(handle, C.byref(limit)) == 0
+        return {
+            "index": index,
+            "name": name.value.decode(),
+            "uuid": uuid.value.decode(),
+            "memory_total_mib": mem.total // _MIB,
+            "power_limit_w": limit.value / 1000 if power_limit else None,
+        }
+
+    def _read_nvlink(self, handle):
+        fields = (_FieldValue * 2)()
+        fields[0].fieldId = _FI_NVLINK_TX
+        fields[1].fieldId = _FI_NVLINK_RX
+        fields[0].scopeId = fields[1].scopeId = _ALL_LINKS
+        if self.lib.nvmlDeviceGetFieldValues(handle, 2, fields) != 0:
+            return None
+        if fields[0].nvmlReturn != 0 or fields[1].nvmlReturn != 0:
+            return None
+        return (fields[0].value.ull, fields[1].value.ull)
+
+    def _probe_nvlink(self, handle):
+        return self._read_nvlink(handle) is not None
+
+    def _get_uint(self, fn, handle, *args):
+        return self._uint.value if fn(handle, *args, C.byref(self._uint)) == 0 else None
+
+    def energy_mj(self, index):
+        # Measured on 8 RTX A6000: 2.5 ms per GPU, about 1 us for the other queries. Read at 50 Hz on 8 GPUs it cost 3.4% of prefill speed
+        # and, worse, under-reported: 818 W where the power gauge said 1,395 W. About twice a second it agrees with the gauge (1,602 W against
+        # 1,591 W). So it is read slowly, and every report compares the two.
+        value = C.c_ulonglong()
+        if self.lib.nvmlDeviceGetTotalEnergyConsumption(self.handles[index], C.byref(value)) == 0:
+            return value.value
+        return None
+
+    def fast(self):
+        lib = self.lib
+        rows = []
+        for handle in self.handles:
+            used = self._mem.used // _MIB if lib.nvmlDeviceGetMemoryInfo(handle, C.byref(self._mem)) == 0 else None
+            power = self._get_uint(lib.nvmlDeviceGetPowerUsage, handle)
+            energy = None
+            sm_clock = self._get_uint(lib.nvmlDeviceGetClockInfo, handle, 1)
+            mem_clock = self._get_uint(lib.nvmlDeviceGetClockInfo, handle, 2)
+            if lib.nvmlDeviceGetUtilizationRates(handle, C.byref(self._util)) == 0:
+                util, util_mem = self._util.gpu, self._util.memory
+            else:
+                util = util_mem = None
+            throttle = None
+            if self._throttle is not None and self._throttle(handle, C.byref(self._ull)) == 0:
+                throttle = self._ull.value
+            temp = self._get_uint(lib.nvmlDeviceGetTemperature, handle, 0)
+            rows.append([used, power / 1000 if power is not None else None, energy, sm_clock, mem_clock, util, util_mem,
+                         throttle, temp])
+        return rows
+
+    def slow(self):
+        rows = []
+        for handle, nvlink_ok in zip(self.handles, self._nvlink_ok):
+            procs = {}
+            if self._procs is not None:
+                count = C.c_uint(_MAX_PROCS)
+                if self._procs(handle, C.byref(count), self._proc_buffer) == 0:
+                    for i in range(min(count.value, _MAX_PROCS)):
+                        info = self._proc_buffer[i]
+                        if info.used != _NOT_AVAILABLE:
+                            procs[info.pid] = info.used // _MIB
+            rows.append({"procs": procs, "nvlink": self._read_nvlink(handle) if nvlink_ok else None})
+        return rows
+
+    def util_samples(self):
+        """New utilization samples per device since the last call, as (wall ns, %). The driver keeps a history at a finer
+        grain than the value a single query returns, so short phases are not judged by one stale reading."""
+        rows = []
+        for i, handle in enumerate(self.handles):
+            count = C.c_uint(len(self._samples))
+            value_type = C.c_int()
+            new = []
+            rc = self.lib.nvmlDeviceGetSamples(handle, 1, C.c_ulonglong(self._util_seen[i]), C.byref(value_type), C.byref(count),
+                                               self._samples)  # NVML_GPU_UTILIZATION_SAMPLES
+            if rc == 0:
+                for k in range(count.value):
+                    sample = self._samples[k]
+                    # The value is a union, and a utilization only sets its low 32 bits.
+                    new.append((sample.timeStamp * 1000, sample.value & 0xFFFFFFFF))
+                if new:
+                    self._util_seen[i] = new[-1][0] // 1000
+            rows.append(new)
+        return rows
+
+    def pcie_kbps(self, index, direction):
+        """PCIe throughput in KB/s, direction 0 = GPU to host, 1 = host to GPU. The driver measures over 20 ms and blocks for
+        that long, so this only ever runs on its own thread, one query at a time."""
+        value = C.c_uint()
+        if self.lib.nvmlDeviceGetPcieThroughput(self.handles[index], direction, C.byref(value)) == 0:
+            return value.value
+        return None
+
+    def close(self):
+        try:
+            self.lib.nvmlShutdown()
+        except OSError:
+            pass
+
+
+def open_backend():
+    """The best backend on this machine. Never raises: no GPU still gives a host-only profile."""
+    if os.environ.get("LLMPERFSCOPE_NO_GPU"):
+        return GpuBackend(reason="LLMPERFSCOPE_NO_GPU is set")
+    try:
+        return NvmlBackend()
+    except (OSError, AttributeError) as e:
+        return GpuBackend(reason=f"NVML not available: {e}")

llmperfscope/html.py

@@ -0,0 +1,320 @@
+"""One self-contained HTML page per run: the lifecycle on a shared time axis, with the diagnosis on top. No scripts, no network,
+nothing to install, so the file can be mailed or attached to a pull request."""
+
+from __future__ import annotations
+
+import html as html_lib
+import json
+from pathlib import Path
+
+from .gpu import G_MEM, G_SM, G_UTIL
+from .sampler import H_ANON, H_FILE, H_STICKS, H_UTICKS, energy_series, load_ticks
+from .system import SECTOR
+from .procfs import CLK_TCK
+
+WIDTH = 1120
+PLOT_LEFT = 74
+PLOT_RIGHT = 16
+ROW_HEIGHT = 118
+GROUP_COLORS = {
+    "startup": "#cbd5e1", "download": "#d8b4fe", "load": "#fcd34d", "setup": "#fdba74", "warmup": "#fda4af", "prefill": "#7dd3fc",
+    "decode": "#86efac", "serve": "#86efac", "run": "#86efac", "observe": "#86efac", "teardown": "#cbd5e1", "idle": "#e2e8f0",
+}
+SERIES_COLORS = ["#2563eb", "#dc2626", "#059669", "#d97706", "#7c3aed", "#0891b2", "#db2777", "#65a30d"]
+
+
+def _escape(text):
+    return html_lib.escape(str(text))
+
+
+def _downsample(points, limit=WIDTH):
+    if len(points) <= limit:
+        return points
+    step = len(points) / limit
+    return [points[int(i * step)] for i in range(limit)]
+
+
+def _scale(values, floor=None):
+    """A round upper bound for an axis."""
+    top = max((v for v in values if v is not None), default=0)
+    if floor is not None:
+        top = max(top, floor)
+    if top <= 0:
+        return 1.0
+    magnitude = 10 ** (len(str(int(top))) - 1) if top >= 1 else 0.1
+    for factor in (1, 2, 2.5, 5, 10):
+        if top <= factor * magnitude:
+            return factor * magnitude
+    return top
+
+
+def _format_axis(value):
+    if value >= 1000:
+        return f"{value / 1000:g}k"
+    return f"{value:g}"
+
+
+def _chart(title, unit, series, phases, t_max, floor=None):
+    """series: [(label, [(t, value), ...])]. Colored bands behind the lines are the phases."""
+    all_values = [v for _, points in series for _, v in points]
+    if not all_values:
+        return ""
+    y_max = _scale(all_values, floor)
+    plot_width = WIDTH - PLOT_LEFT - PLOT_RIGHT
+    x = lambda t: PLOT_LEFT + plot_width * t / max(t_max, 1e-9)  # noqa: E731
+    y = lambda v: 10 + (ROW_HEIGHT - 28) * (1 - min(v, y_max) / y_max)  # noqa: E731
+    parts = [f'<svg viewBox="0 0 {WIDTH} {ROW_HEIGHT}" role="img" aria-label="{_escape(title)}">']
+    for phase in phases:
+        color = GROUP_COLORS.get(phase["group"])
+        if color and phase["group"] not in ("baseline", "tail"):
+            x0, x1 = x(phase["start_s"]), x(phase["end_s"])
+            parts.append(f'<rect x="{x0:.1f}" y="10" width="{max(x1 - x0, 0.5):.1f}" height="{ROW_HEIGHT - 28}" fill="{color}" opacity="0.14"/>')
+    for fraction in (0, 0.5, 1):
+        level = y(y_max * fraction)
+        parts.append(f'<line x1="{PLOT_LEFT}" x2="{WIDTH - PLOT_RIGHT}" y1="{level:.1f}" y2="{level:.1f}" stroke="var(--grid)" stroke-width="1"/>')
+        parts.append(f'<text x="{PLOT_LEFT - 8}" y="{level + 4:.1f}" text-anchor="end" class="tick">{_format_axis(y_max * fraction)}</text>')
+    parts.append(f'<text x="{PLOT_LEFT}" y="{ROW_HEIGHT - 6}" class="tick">{_escape(title)} ({_escape(unit)})</text>')
+    legend_x = WIDTH - PLOT_RIGHT
+    for index, (label, points) in reversed(list(enumerate(series))):
+        color = SERIES_COLORS[index % len(SERIES_COLORS)]
+        points = _downsample(points)
+        path = " ".join(f"{x(t):.1f},{y(v):.1f}" for t, v in points)
+        parts.append(f'<polyline points="{path}" fill="none" stroke="{color}" stroke-width="1.5"/>')
+        if len(series) > 1 or label:
+            parts.append(f'<text x="{legend_x}" y="{ROW_HEIGHT - 6}" text-anchor="end" class="tick" fill="{color}" style="fill:{color}">{_escape(label)}</text>')
+            legend_x -= 8 * len(str(label)) + 16
+    parts.append("</svg>")
+    return "".join(parts)
+
+
+def _rates(ticks, getter, t0, min_dt_s=0.0):
+    """A cumulative counter as a rate per second. Counters that update in steps (the GPU energy counter, CPU ticks) need a window
+    wider than a step, or the rate jumps between zero and a multiple of its true value."""
+    out = []
+    previous = None
+    for tick in ticks:
+        value = getter(tick)
+        if value is None:
+            continue
+        if previous is None:
+            previous = (tick["t"], value)
+            continue
+        dt = (tick["t"] - previous[0]) / 1e9
+        if dt > 0 and dt >= min_dt_s:
+            out.append(((tick["t"] - t0) / 1e9, max(0.0, (value - previous[1]) / dt)))
+            previous = (tick["t"], value)
+    return out
+
+
+def _energy_rate(ticks, device, t0):
+    """Average power between readings of the energy counter."""
+    times, joules = energy_series(ticks, device)
+    points = []
+    anchor = None
+    for t, value in zip(times, joules):
+        if anchor is None:
+            anchor = (t, value)
+        elif t - anchor[0] >= 300_000_000:
+            points.append(((t - t0) / 1e9, max(0.0, (value - anchor[1]) / ((t - anchor[0]) / 1e9))))
+            anchor = (t, value)
+    return points
+
+
+def _table(headers, rows):
+    head = "".join(f"<th>{_escape(h)}</th>" for h in headers)
+    body = "".join("<tr>" + "".join(f"<td>{_escape(c)}</td>" for c in row) + "</tr>" for row in rows)
+    return f"<table><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>"
+
+
+def _gib(mib):
+    return "-" if mib is None else f"{mib / 1024:,.2f}"
+
+
+def _value(value, digits=1):
+    if value is None:
+        return "-"
+    return f"{value:,.{digits}f}" if isinstance(value, float) else f"{value:,}"
+
+
+def render(run, ticks):
+    phases = run.get("phases") or []
+    t0 = ticks[0]["t"] if ticks else 0
+    t_max = (ticks[-1]["t"] - t0) / 1e9 if ticks else 1.0
+    devices = run.get("devices_used") or []
+    out = []
+    title = run.get("label") or run.get("adapter") or "llmperfscope run"
+    out.append(f"<h1>{_escape(title)}</h1>")
+    facts = [f"status: {run.get('status')}", f"adapter: {run.get('adapter')}"]
+    host = (run.get("host") or {}).get("hostname")
+    if host:
+        facts.append(f"host: {host}")
+    before = run.get("machine_before") or {}
+    if "clean" in before:
+        facts.append("machine clean" if before["clean"] else "machine NOT clean")
+    if before.get("model_page_cache_fraction") is not None:
+        facts.append(f"model {before['model_page_cache_fraction'] * 100:.0f}% in page cache")
+    storage = before.get("model_storage") or {}
+    kinds = sorted({d.get("kind", "?") for d in storage.get("disks", [])})
+    if kinds:
+        facts.append(f"model on {'/'.join(kinds)} ({storage.get('fstype')})")
+    out.append(f'<p class="facts">{" · ".join(_escape(f) for f in facts)}</p>')
+    if run.get("perturbed"):
+        out.append('<p class="warn">Traced or sampled run: use it to find where time goes, not for speed numbers.</p>')
+
+    diagnosis = run.get("diagnosis") or {}
+    judged = {name: b for name, b in (diagnosis.get("bottlenecks") or {}).items() if b["confidence"] != "none"}
+    if judged:
+        out.append("<h2>What limits each phase</h2>")
+        out.append(_table(["phase", "limit", "confidence", "why"], [[n, b["limit"], b["confidence"], b["why"]] for n, b in judged.items()]))
+    for heading, key in (("Memory", "memory"), ("Data movement", "movement"), ("Serving", "serving"), ("GPU and host time", "timeline")):
+        findings = diagnosis.get(key) or []
+        if findings:
+            out.append(f"<h2>{heading}</h2><ul>")
+            for finding in sorted(findings, key=lambda f: f["severity"] != "warning"):
+                css = ' class="warn"' if finding["severity"] == "warning" else ""
+                out.append(f"<li{css}>{_escape(finding['text'])}</li>")
+            out.append("</ul>")
+
+    out.append("<h2>Lifecycle</h2>")
+    used_groups = []
+    for phase in phases:
+        if phase["group"] not in used_groups and phase["group"] in GROUP_COLORS:
+            used_groups.append(phase["group"])
+    legend = "".join(f'<span class="swatch" style="background:{GROUP_COLORS[g]}"></span>{_escape(g)} ' for g in used_groups)
+    out.append(f'<p class="legend">{legend}</p>')
+
+    if ticks:
+        t = lambda tick: (tick["t"] - t0) / 1e9  # noqa: E731
+        slow = [tick for tick in ticks if "s" in tick]
+        charts = []
+        if devices:
+            charts.append(_chart("GPU memory", "MiB", [(f"GPU {d}", [(t(k), k["g"][d][G_MEM] or 0) for k in ticks]) for d in devices], phases, t_max))
+            charts.append(_chart("GPU power", "W", [(f"GPU {d}", _energy_rate(ticks, d, t0)) for d in devices], phases, t_max))
+            charts.append(_chart("GPU utilization", "%", [(f"GPU {d}", _util_points(slow, d, ticks, t0)) for d in devices], phases, t_max, floor=100))
+            charts.append(_chart("GPU clock", "MHz", [(f"GPU {d}", [(t(k), k["g"][d][G_SM] or 0) for k in ticks]) for d in devices], phases, t_max))
+        charts.append(_chart("Host memory of the process", "GiB", [
+            ("anonymous", [(t(k), k["h"][H_ANON] / (1 << 20)) for k in ticks]),
+            ("file-backed", [(t(k), k["h"][H_FILE] / (1 << 20)) for k in ticks])], phases, t_max))
+        charts.append(_chart("Host CPU of the process", "cores", [("", _rates(ticks, lambda k: (k["h"][H_UTICKS] + k["h"][H_STICKS]) / CLK_TCK, t0, 0.25))], phases, t_max))
+        disks = sorted({name for k in slow for name in k["s"].get("sys", {}).get("disk", {})})
+        active = []
+        for name in disks:
+            read = _rates(slow, lambda k, n=name: (k["s"].get("sys", {}).get("disk", {}).get(n) or [None])[0], t0)
+            points = [(p[0], p[1] * SECTOR / (1 << 20)) for p in read]
+            if any(v > 0.5 for _, v in points):
+                kind = (run.get("disks") or {}).get(name, {}).get("kind", "disk")
+                active.append((f"{name} ({kind})", points))
+        if active:
+            charts.append(_chart("Disk reads, all processes", "MiB/s", active, phases, t_max))
+        pressure = []
+        for key, label in (("io_some", "I/O"), ("memory_some", "memory"), ("cpu_some", "CPU")):
+            points = [(p[0], min(100.0, p[1] / 1e4)) for p in _rates(slow, lambda k, c=key: k["s"].get("sys", {}).get("psi", {}).get(c), t0)]
+            if any(v > 1 for _, v in points):
+                pressure.append((label, points))
+        if pressure:
+            charts.append(_chart("Time tasks stalled, all processes", "%", pressure, phases, t_max, floor=100))
+        pcie = []
+        for direction, label in ((1, "RAM to GPU"), (0, "GPU to RAM")):
+            points = []
+            for k in slow:
+                rows = k["s"].get("pcie")
+                if rows:
+                    values = [rows[d][direction] for d in devices if rows[d][direction] is not None]
+                    if values:
+                        points.append((t(k), sum(values) / 1024))
+            if points:
+                pcie.append((label, points))
+        if pcie:
+            charts.append(_chart("PCIe traffic, sampled", "MiB/s", pcie, phases, t_max))
+        nvlink = []
+        for d in devices:
+            points = _rates([k for k in slow if k["s"].get("nvlink") and k["s"]["nvlink"][d]], lambda k, d=d: k["s"]["nvlink"][d][0] / 1024, t0)
+            if any(v > 1 for _, v in points):
+                nvlink.append((f"GPU {d}", points))
+        if nvlink:
+            charts.append(_chart("NVLink traffic sent", "MiB/s", nvlink, phases, t_max))
+        out.extend(chart for chart in charts if chart)
+        ticks_text = "".join(f'<span style="left:{PLOT_LEFT / WIDTH * 100 + (WIDTH - PLOT_LEFT - PLOT_RIGHT) / WIDTH * 100 * s / max(t_max, 1e-9):.2f}%">{s:g} s</span>'
+                             for s in _time_marks(t_max))
+        out.append(f'<div class="timeaxis">{ticks_text}</div>')
+
+    rows = []
+    for name, group in (run.get("groups") or {}).items():
+        if name in ("baseline", "tail") or group.get("wall_s", 0.0) < 0.005:
+            continue
+        gpu = group.get("gpu") or {}
+        host_row = group.get("host") or {}
+        rows.append([name, _value(group.get("wall_s"), 2), _value(group.get("tokens")), _gib(gpu.get("mem_end_mib")), _gib(gpu.get("mem_peak_mib")),
+                     _gib(host_row.get("anon_peak_mib")), _gib(host_row.get("file_peak_mib")), _value(gpu.get("energy_j"), 0), _value(gpu.get("util_avg"), 0),
+                     _value(host_row.get("busy_cores"), 2)])
+    if rows:
+        out.append("<h2>Phases</h2>")
+        out.append(_table(["phase", "wall s", "tokens", "GPU GiB end", "GPU GiB peak", "host anon GiB", "host file GiB", "GPU J", "GPU util %", "CPU cores"], rows))
+
+    throughput = run.get("throughput") or {}
+    energy = run.get("energy") or {}
+    rows = []
+    for name, entry in throughput.items():
+        if "tokens_per_s" in entry:
+            rows.append([name, _value(entry["tokens_per_s"], 2), _value(entry.get("stddev"), 2), _value(energy.get(f"{name}_j_per_token"), 3), "-", "-"])
+        elif "output_tokens_per_s" in entry:
+            rows.append([name, _value(entry.get("output_tokens_per_s"), 2), "-", _value(entry.get("j_per_output_token"), 3),
+                         _value((entry.get("ttft_ms") or {}).get("p50"), 1), _value((entry.get("itl_ms") or {}).get("p50"), 1)])
+    if rows:
+        out.append("<h2>Speed</h2>")
+        out.append(_table(["", "tokens/s", "stddev", "GPU J/token", "first token ms p50", "between tokens ms p50"], rows))
+
+    return PAGE.format(title=_escape(title), body="\n".join(out))
+
+
+def _util_points(slow, device, ticks, t0):
+    """The driver's utilization history when there is one, otherwise the instantaneous reading."""
+    points = []
+    for tick in slow:
+        rows = tick["s"].get("util")
+        if rows:
+            points += [((sample_t - t0) / 1e9, value) for sample_t, value in rows[device]]
+    if points:
+        return sorted(p for p in points if p[0] >= 0)
+    return [((tick["t"] - t0) / 1e9, tick["g"][device][G_UTIL] or 0) for tick in ticks]
+
+
+def _time_marks(t_max):
+    step = _scale([t_max / 6])
+    marks = []
+    value = 0.0
+    while value <= t_max:
+        marks.append(round(value, 6))
+        value += step
+    return marks
+
+
+def write(run_dir):
+    run_dir = Path(run_dir)
+    with open(run_dir / "run.json") as f:
+        run = json.load(f)
+    ticks = load_ticks(run_dir / "samples.jsonl.gz")
+    target = run_dir / "report.html"
+    target.write_text(render(run, ticks), encoding="utf-8")
+    return target
+
+
+PAGE = """<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
+<title>{title}</title>
+<style>
+:root {{ --bg:#fff; --fg:#111827; --muted:#6b7280; --grid:#e5e7eb; --warn:#b91c1c; --head:#f3f4f6; }}
+@media (prefers-color-scheme: dark) {{ :root {{ --bg:#0f172a; --fg:#e5e7eb; --muted:#94a3b8; --grid:#1f2937; --warn:#fca5a5; --head:#1e293b; }} }}
+body {{ background:var(--bg); color:var(--fg); font:15px/1.5 -apple-system, "Segoe UI", Roboto, sans-serif; margin:0 auto; padding:24px 16px; max-width:1160px; }}
+h1 {{ font-size:24px; margin:0 0 4px; }} h2 {{ font-size:17px; margin:28px 0 8px; }}
+.facts {{ color:var(--muted); margin:0; }} .legend {{ color:var(--muted); margin:0 0 6px; font-size:13px; }}
+.swatch {{ display:inline-block; width:12px; height:12px; border-radius:2px; margin:0 4px 0 10px; vertical-align:-1px; opacity:.75; }}
+.warn {{ color:var(--warn); }} ul {{ padding-left:20px; margin:0; }} li {{ margin:4px 0; }}
+table {{ border-collapse:collapse; width:100%; font-size:13.5px; display:block; overflow-x:auto; }} th, td {{ padding:5px 10px; text-align:left; border-bottom:1px solid var(--grid); }}
+th {{ background:var(--head); font-weight:600; white-space:nowrap; }}
+svg {{ display:block; width:100%; height:auto; }} .tick {{ font-size:11px; fill:var(--muted); }}
+.timeaxis {{ position:relative; height:20px; font-size:11px; color:var(--muted); }} .timeaxis span {{ position:absolute; transform:translateX(-50%); white-space:nowrap; }}
+</style></head><body>
+{body}
+</body></html>
+"""