llmperfscope 0.1.0__py3-none-any.whl

llmperfscope/compare.py

@@ -0,0 +1,232 @@
+"""A/B comparison. `diff` compares finished runs, `matrix` runs both sides itself in alternating order across many cases.
+
+Speed, memory and energy are judged together, so a change cannot look like a win while it costs memory. A difference only
+counts when the arms do not overlap and it is larger than the noise floor, because a single ratio from a few runs is mostly
+noise: the same binary against itself has shown ratios up to 1.006 at 8 GPUs.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+from . import runner
+
+# name -> (getter, direction): +1 means bigger is better, -1 smaller is better.
+METRICS = {
+    "prefill_tps": (lambda r: _get(r, "throughput", "prefill", "tokens_per_s"), +1),
+    "decode_tps": (lambda r: _get(r, "throughput", "decode", "tokens_per_s"), +1),
+    "gpu_peak_mib": (lambda r: _get(r, "memory", "gpu", "peak_above_baseline_total_mib"), -1),
+    "gpu_peak_max_device_mib": (lambda r: _get(r, "memory", "gpu", "peak_above_baseline_max_device_mib"), -1),
+    "host_anon_peak_mib": (lambda r: _get(r, "memory", "host", "anon_peak_mib"), -1),
+    "load_s": (lambda r: _get(r, "groups", "load", "wall_s"), -1),
+    "prefill_j_per_token": (lambda r: _get(r, "energy", "prefill_j_per_token"), -1),
+    "decode_j_per_token": (lambda r: _get(r, "energy", "decode_j_per_token"), -1),
+}
+# Smallest difference that can mean anything, per metric: relative, then absolute. Memory rounds to whole MiB and allocators
+# differ by a little from run to run; load time depends on the page cache; energy has its own counter resolution.
+FLOORS = {
+    "prefill_tps": (0.003, 0.0),
+    "decode_tps": (0.003, 0.0),
+    "gpu_peak_mib": (0.002, 16.0),
+    "gpu_peak_max_device_mib": (0.002, 16.0),
+    "host_anon_peak_mib": (0.02, 64.0),
+    "load_s": (0.05, 0.5),
+    "prefill_j_per_token": (0.05, 0.0),
+    "decode_j_per_token": (0.03, 0.0),
+}
+
+
+def _get(run, *keys):
+    node = run
+    for key in keys:
+        if not isinstance(node, dict) or key not in node:
+            return None
+        node = node[key]
+    return node
+
+
+def judge(metric, a_values, b_values, noise=0.0):
+    """Verdict for B against A: better, worse or same. `noise` is a relative floor measured with a null comparison."""
+    a_values = [v for v in a_values if v is not None]
+    b_values = [v for v in b_values if v is not None]
+    if not a_values or not b_values:
+        return {"verdict": "missing"}
+    a_mean = sum(a_values) / len(a_values)
+    b_mean = sum(b_values) / len(b_values)
+    direction = METRICS[metric][1]
+    ratio = b_mean / a_mean if a_mean else None
+    relative_floor, absolute_floor = FLOORS[metric]
+    floor = max(relative_floor, noise) * abs(a_mean)
+    floor = max(floor, absolute_floor)
+    difference = (b_mean - a_mean) * direction  # positive: B is better
+    separated = min(b_values) > max(a_values) or max(b_values) < min(a_values)
+    verdict = "same"
+    if abs(difference) > floor and separated:
+        verdict = "better" if difference > 0 else "worse"
+    return {"a": round(a_mean, 4), "b": round(b_mean, 4), "ratio": round(ratio, 4) if ratio is not None else None,
+            "verdict": verdict, "a_runs": a_values, "b_runs": b_values}
+
+
+def diff(runs_a, runs_b, noise=None):
+    """Runs of side A against runs of side B. Failed runs and traced runs do not count; they are listed instead."""
+    usable_a = [r for r in runs_a if r.get("status") == "ok" and not r.get("perturbed")]
+    usable_b = [r for r in runs_b if r.get("status") == "ok" and not r.get("perturbed")]
+    out = {"metrics": {}, "dropped_runs": (len(runs_a) - len(usable_a)) + (len(runs_b) - len(usable_b))}
+    dirty = sum(1 for r in usable_a + usable_b if not (r.get("machine_before") or {"clean": True}).get("clean", True))
+    if dirty:
+        out["dirty_runs"] = dirty
+    for metric, (getter, _direction) in METRICS.items():
+        result = judge(metric, [getter(r) for r in usable_a], [getter(r) for r in usable_b], (noise or {}).get(metric, 0.0))
+        if result["verdict"] != "missing":
+            out["metrics"][metric] = result
+    worse = [m for m, v in out["metrics"].items() if v["verdict"] == "worse"]
+    better = [m for m, v in out["metrics"].items() if v["verdict"] == "better"]
+    out["worse"], out["better"] = worse, better
+    if worse and better:
+        out["verdict"] = "tradeoff"
+    else:
+        out["verdict"] = "regression" if worse else "improvement" if better else "neutral"
+    return out
+
+
+def load_run(path):
+    path = Path(path)
+    if path.is_dir():
+        path = path / "run.json"
+    with open(path) as f:
+        return json.load(f)
+
+
+def render_diff(result, a_label="A", b_label="B"):
+    lines = [f"| metric | {a_label} | {b_label} | ratio | |", "|---|---:|---:|---:|---|"]
+    for metric, v in result["metrics"].items():
+        mark = {"better": "better", "worse": "**worse**", "same": ""}[v["verdict"]]
+        lines.append(f"| {metric} | {v['a']:,} | {v['b']:,} | {v['ratio']} | {mark} |")
+    lines.append(f"\nVerdict: **{result['verdict']}**")
+    if result["dropped_runs"]:
+        lines.append(f"{result['dropped_runs']} failed or traced runs were left out.")
+    if result.get("dirty_runs"):
+        lines.append(f"{result['dirty_runs']} runs started on a machine that was not clean.")
+    return "\n".join(lines) + "\n"
+
+
+def load_cases(path):
+    with open(path) as f:
+        spec = json.load(f)
+    defaults = spec.get("defaults", {})
+    cases = []
+    for case in spec["cases"]:
+        merged = {
+            "label": case["label"],
+            "binary": case.get("binary", defaults.get("binary", "llama-bench")),
+            "prefix": case.get("prefix", defaults.get("prefix", [])),
+            "args": list(defaults.get("args", [])) + list(case.get("args", [])),
+            "env": {**defaults.get("env", {}), **case.get("env", {})},
+            "model": case.get("model"),
+            "gpus": case.get("gpus"),
+        }
+        cases.append(merged)
+    return cases
+
+
+def _command(case, bin_dir):
+    command = list(case["prefix"]) + [str(Path(bin_dir) / case["binary"])]
+    if case["model"]:
+        command += ["-m", case["model"]]
+    return command + case["args"]
+
+
+def _tag(label):
+    return re.sub(r"[^A-Za-z0-9_.-]+", "_", label)
+
+
+def run_matrix(cases, a, b, out_dir, order="ABBA", timeout_s=1800.0, a_env=None, b_env=None, only=None, noise=None, log=print,
+               precheck="wait"):
+    """a and b are (name, bin_dir). Fresh process per arm, alternating order, machine re-checked before every arm."""
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    table = {"a": a[0], "b": b[0], "order": order, "cases": []}
+    for case in cases:
+        if only and not re.search(only, case["label"]):
+            continue
+        if case["model"] and not Path(case["model"]).exists():
+            log(f"{case['label']}: model missing, skipped")
+            table["cases"].append({"label": case["label"], "skipped": "model missing"})
+            continue
+        runs = {"A": [], "B": []}
+        for i, side in enumerate(order):
+            name, bin_dir = a if side == "A" else b
+            env = dict(case["env"])
+            env.update((a_env if side == "A" else b_env) or {})
+            if case["gpus"] is not None:
+                env["CUDA_VISIBLE_DEVICES"] = case["gpus"]
+            options = runner.Options(out=str(out_dir / _tag(case["label"]) / f"{i}_{side}_{name}"), label=f"{case['label']} [{name}]",
+                                     env=env, timeout_s=timeout_s, quiet=True, precheck=precheck)
+            log(f"{case['label']}: arm {i + 1}/{len(order)} ({name})")
+            try:
+                report = runner.Run(_command(case, bin_dir), options).execute()
+            except runner.Refused as e:
+                log(f"  refused: {e}")
+                continue
+            if report["status"] != "ok":
+                log(f"  arm ended with status {report['status']}")
+            runs[side].append(report)
+        result = diff(runs["A"], runs["B"], (noise or {}).get(case["label"]))
+        result["label"] = case["label"]
+        table["cases"].append(result)
+        with open(out_dir / "matrix.json", "w") as f:
+            json.dump(table, f, indent=1)
+        write_table(table, out_dir / "table.md")
+        log(f"  -> {result['verdict']}: " + ", ".join(f"{m} {v['ratio']}" for m, v in result["metrics"].items()
+                                                      if m in ("prefill_tps", "decode_tps")))
+    return table
+
+
+def write_table(table, path):
+    columns = [("prefill_tps", "pp"), ("decode_tps", "tg"), ("gpu_peak_mib", "GPU mem"), ("host_anon_peak_mib", "host anon"),
+               ("load_s", "load"), ("decode_j_per_token", "tg J/tok")]
+    header = "| case | " + " | ".join(f"{title} {table['b']}/{table['a']}" for _, title in columns) + " | verdict |"
+    lines = [header, "|---|" + "---:|" * len(columns) + "---|"]
+    for case in table["cases"]:
+        if "skipped" in case:
+            lines.append(f"| {case['label']} | {case['skipped']} |")
+            continue
+        cells = []
+        for metric, _ in columns:
+            value = case["metrics"].get(metric)
+            if not value:
+                cells.append("-")
+                continue
+            cell = f"{value['ratio']:.3f}" if value["ratio"] is not None else "-"
+            if value["verdict"] == "worse":
+                cell = f"**{cell}** worse"
+            elif value["verdict"] == "better":
+                cell = f"{cell} better"
+            cells.append(cell)
+        extra = f" ({case['dropped_runs']} arms failed)" if case["dropped_runs"] else ""
+        if case.get("worse"):
+            extra += ": costs " + ", ".join(case["worse"])
+        lines.append(f"| {case['label']} | " + " | ".join(cells) + f" | {case['verdict']}{extra} |")
+    with open(path, "w") as f:
+        f.write("\n".join(lines) + "\n")
+
+
+def noise_from_null(matrix_path):
+    """Noise floors from a null comparison (same binary on both sides): per case and metric, the spread that appeared with
+    nothing changed. Pass the file to later matrices with --noise."""
+    with open(matrix_path) as f:
+        table = json.load(f)
+    out = {}
+    for case in table["cases"]:
+        if "metrics" not in case:
+            continue
+        floors = {}
+        for metric, value in case["metrics"].items():
+            values = value["a_runs"] + value["b_runs"]
+            mean = sum(values) / len(values)
+            if mean:
+                floors[metric] = round((max(values) - min(values)) / abs(mean), 5)
+        out[case["label"]] = floors
+    return out

llmperfscope/container.py

@@ -0,0 +1,79 @@
+"""`docker run ...` as a target. The docker client is only a remote control: the engine is a child of the daemon, so the client's
+process tree holds nothing worth measuring. llmperfscope asks docker for the container's first process and follows that instead.
+Memory, CPU and per-process GPU memory all stay visible from the host, because a container's processes are host processes."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import threading
+import time
+from pathlib import Path
+
+
+class DockerTarget:
+    def __init__(self, tool, run_dir):
+        self.tool = tool
+        self.run_dir = Path(run_dir).resolve()
+        self.cidfile = self.run_dir / "container.id"
+        self.container = None
+        self.pid = None
+        self.image = None
+        self._done = threading.Event()
+
+    def prepare(self, cmd, index, marker_file):
+        """Adds what llmperfscope needs right after `docker run`: the id file, and the marker file for scripts inside the container."""
+        if self.cidfile.exists():
+            self.cidfile.unlink()
+        extra = ["--cidfile", str(self.cidfile), "-e", "LLMPERFSCOPE_MARKERS=/llmperfscope-run/markers.log", "-v", f"{self.run_dir}:/llmperfscope-run"]
+        return cmd[:index + 2] + extra + cmd[index + 2:]
+
+    def follow(self, on_pid, timeout_s=300.0):
+        """Finds the container's first process as soon as docker has one, in the background."""
+
+        def wait():
+            deadline = time.monotonic() + timeout_s
+            while time.monotonic() < deadline and not self._done.is_set():
+                if self.cidfile.exists():
+                    self.container = self.cidfile.read_text().strip() or None
+                if self.container:
+                    done = subprocess.run([self.tool, "inspect", "-f", "{{.State.Pid}}", self.container], capture_output=True, text=True,
+                                          check=False)
+                    value = done.stdout.strip()
+                    if value.isdigit() and int(value) > 0:
+                        self.pid = int(value)
+                        on_pid(self.pid)
+                        image = subprocess.run([self.tool, "inspect", "-f", "{{.Config.Image}} {{.Image}}", self.container],
+                                               capture_output=True, text=True, check=False)
+                        self.image = image.stdout.strip()
+                        return
+                time.sleep(0.2)
+
+        threading.Thread(target=wait, daemon=True, name="llmperfscope-container").start()
+
+    def identity(self):
+        """Which image ran, as the report's record of which code this was."""
+        if not self.image:
+            return {}
+        name, _, digest = self.image.partition(" ")
+        return {"container_image": name, "container_image_id": digest}
+
+    def stop(self, wait_s=30):
+        """Stops the container itself. A signal to the client only reaches it while the client is attached."""
+        self._done.set()
+        if self.container:
+            subprocess.run([self.tool, "stop", "-t", str(int(wait_s)), self.container], capture_output=True, text=True, check=False)
+
+    def remove_if_running(self):
+        self._done.set()
+        if not self.container:
+            return False
+        done = subprocess.run([self.tool, "inspect", "-f", "{{.State.Running}}", self.container], capture_output=True, text=True, check=False)
+        if done.stdout.strip() == "true":
+            subprocess.run([self.tool, "kill", self.container], capture_output=True, text=True, check=False)
+            return True
+        return False
+
+
+def tool_name(cmd, index):
+    return os.path.basename(cmd[index])

llmperfscope/cpuprof.py

@@ -0,0 +1,238 @@
+"""Host CPU hot spots without perf: builds and loads native/cpuprof.c, then symbolizes its samples per phase group."""
+
+from __future__ import annotations
+
+import bisect
+import glob
+import os
+import shutil
+import struct
+import subprocess
+import tempfile
+from collections import Counter, defaultdict
+from pathlib import Path
+
+NATIVE = Path(__file__).parent / "native"
+MAGIC = 0x4c5053434f504531
+
+
+def library_path():
+    """Builds the shim next to the package, or in the user's cache when the package directory is read-only."""
+    source = NATIVE / "cpuprof.c"
+    compiler = None
+    for directory in (NATIVE, Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "llmperfscope"):
+        target = directory / "libllmperfscope_cpu.so"
+        if target.exists() and target.stat().st_mtime >= source.stat().st_mtime:
+            return target
+        compiler = compiler or shutil.which("cc") or shutil.which("gcc") or shutil.which("clang")
+        if not compiler:
+            raise FileNotFoundError("the CPU profiler needs a C compiler to build its shim once")
+        try:
+            directory.mkdir(parents=True, exist_ok=True)
+            with tempfile.NamedTemporaryFile(dir=directory, suffix=".so", delete=False) as tmp:
+                temporary = tmp.name
+        except OSError:
+            continue
+        subprocess.run([compiler, "-O2", "-g", "-shared", "-fPIC", "-pthread", "-o", temporary, str(source), "-ldl", "-lrt"], check=True)
+        os.replace(temporary, target)
+        return target
+    raise OSError("no writable directory for the CPU profiler shim")
+
+
+def prepare_env(env, run_dir, hz):
+    env = dict(env)
+    library = str(library_path())
+    existing = env.get("LD_PRELOAD")
+    env["LD_PRELOAD"] = f"{library}:{existing}" if existing else library
+    env["LLMPERFSCOPE_CPUPROF_OUT"] = str(Path(run_dir).resolve() / "cpuprof")
+    env["LLMPERFSCOPE_CPUPROF_HZ"] = str(hz)
+    return env
+
+
+class _Symbols:
+    """Address -> function for one mapped file, from its ELF symbol tables."""
+
+    def __init__(self, path):
+        self.path = path
+        self.starts = []
+        self.ends = []
+        self.names = []
+        self.segments = []
+        if os.path.exists(path):
+            self.segments = _load_segments(path)
+            pairs = {}
+            nm = shutil.which("nm")
+            if nm:
+                for extra in ([], ["-D"]):
+                    done = subprocess.run([nm, "-C", "-S", "--defined-only", *extra, path], capture_output=True, text=True,
+                                          check=False, errors="replace")
+                    for line in done.stdout.splitlines():
+                        parts = line.split(None, 3)
+                        if len(parts) == 4 and parts[2] in "tTwWiI":
+                            try:
+                                pairs.setdefault(int(parts[0], 16), (int(parts[1], 16), parts[3]))
+                            except ValueError:
+                                pass
+            for address in sorted(pairs):
+                self.starts.append(address)
+                self.ends.append(address + pairs[address][0])
+                self.names.append(pairs[address][1])
+
+    def lookup(self, file_offset):
+        """Only an address inside a symbol's extent gets its name. In a stripped library such as the GPU driver, the nearest
+        exported symbol before an address says nothing about the code there, and printing it would mislead."""
+        address = file_offset
+        for p_offset, p_vaddr, p_filesz in self.segments:
+            if p_offset <= file_offset < p_offset + p_filesz:
+                address = file_offset - p_offset + p_vaddr
+                break
+        i = bisect.bisect_right(self.starts, address) - 1
+        if i >= 0 and address < self.ends[i]:
+            return self.names[i]
+        return None
+
+
+def _load_segments(path):
+    """PT_LOAD program headers of a 64-bit ELF: maps file offsets to the addresses nm prints."""
+    try:
+        with open(path, "rb") as f:
+            header = f.read(64)
+            if header[:4] != b"\x7fELF" or header[4] != 2:
+                return []
+            e_phoff = struct.unpack_from("<Q", header, 32)[0]
+            e_phentsize, e_phnum = struct.unpack_from("<HH", header, 54)
+            f.seek(e_phoff)
+            table = f.read(e_phentsize * e_phnum)
+    except OSError:
+        return []
+    segments = []
+    for i in range(e_phnum):
+        base = i * e_phentsize
+        p_type = struct.unpack_from("<I", table, base)[0]
+        if p_type == 1:
+            p_offset, p_vaddr = struct.unpack_from("<QQ", table, base + 8)
+            p_filesz = struct.unpack_from("<Q", table, base + 32)[0]
+            segments.append((p_offset, p_vaddr, p_filesz))
+    return segments
+
+
+def _load_maps(path):
+    ranges = []
+    with open(path) as f:
+        for line in f:
+            parts = line.split(None, 5)
+            if len(parts) == 6 and "x" in parts[1]:
+                start, end = (int(x, 16) for x in parts[0].split("-"))
+                ranges.append((start, end, int(parts[2], 16), parts[5].strip()))
+    ranges.sort()
+    return ranges
+
+
+def _read_samples(path):
+    """[(time ns, tid, thread name, frames)], dropped count, samples per CPU second."""
+    with open(path, "rb") as f:
+        magic, count, dropped, max_frames, hz = struct.unpack("<5Q", f.read(40))
+        if magic != MAGIC:
+            raise ValueError("not a llmperfscope CPU profile")
+        size = 32 + 8 * max_frames
+        layout = f"<QII16s{max_frames}Q"
+        samples = []
+        for _ in range(count):
+            raw = f.read(size)
+            if len(raw) < size:
+                break
+            values = struct.unpack(layout, raw)
+            samples.append((values[0], values[1], values[3].split(b"\0", 1)[0].decode("utf-8", "replace"), values[4:4 + values[2]]))
+    return samples, dropped, hz
+
+
+def summarize(run_dir, phases, t_baseline_ns, top=20):
+    """Per phase group: where the CPU samples are, by module, function and thread. For the hot functions it also says how many threads
+    ran them, which is what tells serial code from parallel code."""
+    files = sorted(glob.glob(str(Path(run_dir) / "cpuprof.*.bin")))
+    if not files:
+        return {"error": "no samples: the process was killed before it could write them, or ran no CPU"}
+    starts = [t_baseline_ns + int(p["start_s"] * 1e9) for p in phases]
+    groups = [p["group"] for p in phases]
+    wall_by_group = Counter()
+    for phase in phases:
+        wall_by_group[phase["group"]] += phase["end_s"] - phase["start_s"]
+    symbols = {}
+    per_group = defaultdict(lambda: {"n": 0, "modules": Counter(), "self": Counter(), "inclusive": Counter(), "threads": Counter(),
+                                     "per_tid": Counter(), "self_tids": defaultdict(Counter), "tid_names": {}})
+    total = dropped_total = 0
+    period_s = 0.005
+    for path in files:
+        ranges = _load_maps(path[:-4] + ".maps")
+        range_starts = [r[0] for r in ranges]
+        samples, dropped, hz = _read_samples(path)
+        dropped_total += dropped
+        period_s = 1.0 / hz if hz else period_s
+
+        def resolve(address):
+            i = bisect.bisect_right(range_starts, address) - 1
+            if i < 0 or address >= ranges[i][1]:
+                return "[unknown]", "[unknown]"
+            start, _end, offset, module_path = ranges[i]
+            module = os.path.basename(module_path) or "[anon]"
+            if not module_path.startswith("/"):
+                return module, module
+            table = symbols.get(module_path)
+            if table is None:
+                table = symbols[module_path] = _Symbols(module_path)
+            name = table.lookup(address - start + offset)
+            return module, name if name else f"{module}!?"
+
+        cache = {}
+        for t, tid, comm, frames in samples:
+            if not frames:
+                continue
+            total += 1
+            i = bisect.bisect_right(starts, t) - 1
+            bucket = per_group[groups[i] if i >= 0 else "before"]
+            bucket["n"] += 1
+            bucket["threads"][_thread_label(comm)] += 1
+            bucket["per_tid"][tid] += 1
+            bucket["tid_names"][tid] = comm
+            seen = set()
+            for depth, address in enumerate(frames):
+                resolved = cache.get(address)
+                if resolved is None:
+                    resolved = cache[address] = resolve(address)
+                module, name = resolved
+                if depth == 0:
+                    bucket["self"][name] += 1
+                    bucket["modules"][module] += 1
+                    bucket["self_tids"][name][tid] += 1
+                if name not in seen:
+                    seen.add(name)
+                    bucket["inclusive"][name] += 1
+    out = {"samples": total, "dropped": dropped_total, "period_s": period_s, "groups": {}}
+    for group, bucket in per_group.items():
+        n = bucket["n"] or 1
+        wall = wall_by_group.get(group, 0.0)
+        self_rows = []
+        for name, count in bucket["self"].most_common(top):
+            tids = bucket["self_tids"][name]
+            busiest = max(tids.values())
+            row = {"name": name[:96], "share": round(count / n, 4), "cpu_s": round(count * period_s, 3), "threads": len(tids),
+                   # With all threads in this function at once, the time it holds up the run is the longest any one thread spent in it.
+                   "busiest_thread_s": round(busiest * period_s, 3)}
+            if wall > 0:
+                row["pct_of_wall_on_busiest_thread"] = round(100 * busiest * period_s / wall, 1)
+            self_rows.append(row)
+        out["groups"][group] = {
+            "samples": bucket["n"],
+            "cpu_s": round(bucket["n"] * period_s, 3),
+            "modules": [{"name": k, "share": round(v / n, 4)} for k, v in bucket["modules"].most_common(8)],
+            "self": self_rows,
+            "inclusive": [{"name": k[:96], "share": round(v / n, 4)} for k, v in bucket["inclusive"].most_common(top)],
+            "threads": [{"name": k, "share": round(v / n, 4)} for k, v in bucket["threads"].most_common(6)],
+            "busiest_threads": [{"tid": tid, "name": bucket["tid_names"][tid], "cpu_s": round(count * period_s, 3)}
+                                for tid, count in bucket["per_tid"].most_common(8)],
+        }
+    return out
+
+
+def _thread_label(name):
+    return "".join("#" if c.isdigit() else c for c in name)