inference-vitals 0.2.0__py3-none-any.whl

inference_vitals/conditions.py

@@ -0,0 +1,63 @@
+"""What the Mac was doing when a run happened, so a slow page can be told apart from a hot or busy Mac.
+
+Read from macOS's own commands (pmset, sysctl, sw_vers, ps), none of which need sudo. Process names are
+kept, never their arguments, since arguments can hold file names.
+"""
+
+import re
+import subprocess
+import time
+
+MODES = {"0": "normal", "1": "low power", "2": "high power"}
+
+
+def _out(*cmd) -> str:
+    """A command's output, or "" when the command does not exist (macOS tools on Linux)."""
+    try:
+        return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
+    except FileNotFoundError:
+        return ""
+
+
+def thermal(text: str) -> dict:
+    def level(pattern, default):
+        m = re.search(pattern, text, re.IGNORECASE)
+        return int(m.group(1)) if m else default
+
+    return {
+        "thermal_warning": level(r"thermal warning level set to (\d+)", 0),
+        "performance_warning": level(r"performance warning level set to (\d+)", 0),
+        "cpu_speed_limit": level(r"CPU_Speed_Limit\s*=\s*(\d+)", 100),
+    }
+
+
+def on_mains(batt: str) -> bool:
+    return "'AC Power'" in batt
+
+
+def power_mode(pm: str) -> str:
+    m = re.search(r"\bpowermode\s+(\d)", pm)
+    return MODES.get(m.group(1), "unknown") if m else "unknown"
+
+
+def busy_processes(ps: str, over: float = 20.0) -> list:
+    busy = []
+    for line in ps.splitlines():
+        m = re.match(r"\s*([\d.]+)\s+(.+)$", line)
+        if m and float(m.group(1)) >= over:
+            busy.append([m.group(2).rstrip().rsplit("/", 1)[-1], float(m.group(1))])
+    return busy
+
+
+def snapshot() -> dict:
+    limit = _out("sysctl", "-n", "iogpu.wired_limit_mb").strip()
+    return {
+        "at": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
+        "macos": _out("sw_vers", "-productVersion").strip(),
+        "model": _out("sysctl", "-n", "hw.model").strip(),
+        "on_mains": on_mains(_out("pmset", "-g", "batt")),
+        "power_mode": power_mode(_out("pmset", "-g")),
+        **thermal(_out("pmset", "-g", "therm")),
+        "gpu_wired_limit_mb": int(limit) if limit.isdigit() else None,
+        "busy_processes": busy_processes(_out("ps", "-Ao", "pcpu=,comm=", "-r")),
+    }

inference_vitals/follow.py

@@ -0,0 +1,196 @@
+"""Follow a fine-tuning run from what its trainer prints, and publish it for the dashboard. The trainer
+needs no change.
+
+  inference-vitals train -- python -m mlx_vlm.train --model-path ... --data ...   wrap the command
+  inference-vitals watch-training train.log                                     follow a log file
+
+Read: mlx-vlm's trainer (metric blocks such as "Training (iteration 10):" then name=value pairs, its
+progress bar, and "Saved adapter weights" lines) and mlx-lm's ("Iter 10: Train loss 1.23, ..."). Lines
+it does not recognise change nothing. A trainer of your own can publish directly with
+inference_vitals.training instead.
+"""
+
+import argparse
+import codecs
+import os
+import re
+import subprocess
+import sys
+import time
+
+from inference_vitals import training
+from inference_vitals.metrics import DEFAULT_PORT
+
+ANSI = re.compile(r"\x1b\[[0-9;]*m")
+TOTAL = re.compile(r"Starting training\.\.\., (?:iterations|iters): (\d+)")
+HEADER = re.compile(r"^(Training|Validation) \(iteration (\d+)\):")
+PAIR = re.compile(r"\b([a-z_]+)=(-?[\d,]*\.?\d+(?:e[-+]?\d+)?)")
+BAR = re.compile(r"\|\s*(\d+)/(\d+) \[[^\]]*?loss=([\d.]+)(?:, lr=([\d.e+-]+))?")
+SAVED = re.compile(r"Saved (?:final )?(?:adapter|trainable) weights")  # mlx-lm; mlx-vlm 0.7
+LM_TRAIN = re.compile(
+    r"Iter (\d+): Train loss ([\d.]+), Learning Rate ([\d.e+-]+).*?Tokens/sec ([\d.]+)"
+    r"(?:, Trained Tokens (\d+))?.*?Peak mem ([\d.]+) GB"
+)
+LM_VAL = re.compile(r"Iter (\d+): Val loss ([\d.]+)")
+KEYS = {  # mlx-vlm's names -> training.Training.step's
+    "train_loss": "loss",
+    "learning_rate": "learning_rate",
+    "tokens_per_second": "tokens_per_second",
+    "peak_memory": "peak_memory_gb",
+    "trained_tokens": "trained_tokens",
+    "step_time": "step_seconds",  # its own timing: lines arrive in bursts, so timing them would mislead
+}
+
+
+class Reader:
+    """Turns a trainer's printed lines into calls on a training.Training run."""
+
+    def __init__(self, run: training.Training):
+        self.run, self.block, self.step = run, None, 0
+
+    def feed(self, text: str):
+        for part in ANSI.sub("", text).split("\r"):
+            self._part(part.strip())
+
+    def _part(self, s: str):
+        if not s:
+            return
+        run = self.run
+        if m := TOTAL.search(s):
+            run.total_steps = int(m.group(1))
+        elif m := HEADER.match(s):
+            self.block, self.step = m.group(1), int(m.group(2))
+            if self.block == "Validation":
+                run.validating()
+        elif SAVED.search(s):
+            run.saving()
+        elif m := LM_VAL.search(s):
+            run.validating()
+            run.validate(float(m.group(2)))
+        elif m := LM_TRAIN.search(s):
+            step, loss, lr, tps, tokens, mem = m.groups()
+            run.step(int(step), loss=float(loss), learning_rate=float(lr), tokens_per_second=float(tps),
+                     peak_memory_gb=float(mem), trained_tokens=int(tokens) if tokens else None)  # fmt: skip
+        elif m := BAR.search(s):
+            step, total, loss, lr = m.groups()
+            run.total_steps = run.total_steps or int(total)
+            run.step(int(step), loss=float(loss), learning_rate=float(lr) if lr else None)
+        elif self.block and (pairs := PAIR.findall(s)):
+            nums = {k: float(v.replace(",", "")) for k, v in pairs}
+            if self.block == "Validation":
+                if "val_loss" in nums:
+                    run.validate(nums["val_loss"])
+            elif mapped := {KEYS[k]: v for k, v in nums.items() if k in KEYS}:
+                run.step(self.step, **mapped)
+
+
+def _parts(chunks):
+    """The pieces of text between line ends and carriage returns, as chunks arrive."""
+    buf = ""
+    for chunk in chunks:
+        buf += chunk
+        *done, buf = re.split(r"[\r\n]", buf)
+        yield from done
+    if buf:
+        yield buf
+
+
+def train(
+    cmd: list[str], name: str, port: int = DEFAULT_PORT, epochs=None, on_finish=None, linger=20.0
+) -> int:
+    """Run the trainer, pass its output through untouched, and publish what it reports while it runs."""
+    run = training.start(name, epochs=epochs, port=port)
+    reader = Reader(run)
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+                            env={**os.environ, "PYTHONUNBUFFERED": "1"})  # fmt: skip
+
+    decode = codecs.getincrementaldecoder("utf-8")(errors="replace").decode  # a character may span two reads
+
+    def chunks():
+        while data := os.read(proc.stdout.fileno(), 4096):
+            sys.stdout.buffer.write(data)  # the trainer's own bytes, untouched
+            sys.stdout.buffer.flush()
+            yield decode(data)
+
+    try:
+        for part in _parts(chunks()):
+            reader.feed(part)
+        return proc.wait()
+    except KeyboardInterrupt:  # Ctrl-C: stop the trainer too, and do not wait long for the last read
+        proc.terminate()
+        proc.wait(10)
+        linger = min(linger, 5)
+        return 130
+    finally:
+        run.finish(linger)
+        if on_finish:
+            on_finish(run)
+
+
+IDLE_AFTER = 300  # seconds without a new line before a watched run counts as no longer training
+
+
+def watch(
+    path, run: training.Training, stop=lambda: False, poll=1.0, clock=time.monotonic, idle_after=IDLE_AFTER
+):
+    """Read a log file as it grows, until stop() says so at its end (Ctrl-C from the command line). A log
+    that stops growing for `idle_after` seconds marks the run idle, so it is not shown training for days."""
+    reader = Reader(run)
+    with open(path, errors="replace") as f:
+
+        def chunks():
+            last = clock()
+            while True:
+                text = f.read()
+                if text:
+                    last = clock()
+                    yield text
+                elif stop():
+                    return
+                else:
+                    if clock() - last > idle_after and run.phase != training.PHASES["idle"]:
+                        run.idle()
+                    time.sleep(poll)
+
+        for part in _parts(chunks()):
+            reader.feed(part)
+
+
+def main(action: str, argv: list[str]):
+    p = argparse.ArgumentParser(prog=f"inference-vitals {action}", description=__doc__,
+                                formatter_class=argparse.RawDescriptionHelpFormatter)  # fmt: skip
+    p.add_argument("--name", help="the run's name on the dashboard (default: from the command or file)")
+    p.add_argument("--epochs", type=int)
+    p.add_argument("--port", type=int, default=DEFAULT_PORT, help="where the dashboard reads numbers")
+    if action == "train":
+        own, _, cmd = _split(argv)
+        args = p.parse_args(own)
+        if not cmd:
+            p.error("give the training command after --")
+        return train(cmd, args.name or default_name(cmd), args.port, args.epochs)
+    p.add_argument("log", help="the trainer's log file")
+    args = p.parse_args(argv)
+    run = training.start(args.name or os.path.basename(args.log), epochs=args.epochs, port=args.port)
+    try:
+        watch(args.log, run)
+    except KeyboardInterrupt:
+        pass
+    finally:
+        run.finish()
+    return 0
+
+
+def default_name(cmd: list[str]) -> str:
+    """A run's name when none is given: the module after -m, else the script, else the program."""
+    if "-m" in cmd[:-1]:
+        return cmd[cmd.index("-m") + 1]
+    script = next((a for a in cmd[1:] if a.endswith(".py")), None)
+    return os.path.basename(script or cmd[0])
+
+
+def _split(argv: list[str]):
+    """(our options, "--", the trainer's command)."""
+    if "--" in argv:
+        i = argv.index("--")
+        return argv[:i], "--", argv[i + 1 :]
+    return argv, "", []

inference_vitals/gpustat.py

@@ -0,0 +1,147 @@
+"""NVIDIA GPUs as Prometheus numbers on 127.0.0.1:9091 (started by inference-vitals on Linux), read from
+`nvidia-smi`, which comes with every NVIDIA driver. No sudo, nothing else to install.
+
+Each GPU gets its own labels (gpu="0", name="NVIDIA GeForce RTX 4090"). A value a GPU does not report
+("[N/A]", common for power and fan on laptops and some server cards) is left out, not sent as zero.
+
+Slowdown state, the GPU's own reasons for running below full clock speed, worst first:
+  0 none · 1 power cap (normal under full load) · 2 thermal slowdown · 3 hardware slowdown (power brake,
+  external signal). Newer drivers call these `clocks_event_reasons`, older ones `clocks_throttle_reasons`;
+  both are tried.
+
+  python -m inference_vitals.gpustat --port 9091
+"""
+
+import argparse
+import shutil
+import subprocess
+
+from inference_vitals.metrics import labels as label_text
+
+FIELDS = [
+    "index",
+    "name",
+    "utilization.gpu",
+    "memory.total",
+    "memory.used",
+    "memory.free",
+    "temperature.gpu",
+    "power.draw",
+    "power.limit",
+    "clocks.gr",
+    "clocks.mem",
+    "fan.speed",
+]
+MIB = 2**20
+# field -> (metric, multiplier)
+NUMBERS = {
+    "utilization.gpu": ("vitals_gpu_busy_ratio", 0.01),
+    "memory.total": ("vitals_gpu_memory_total_bytes", MIB),
+    "memory.used": ("vitals_gpu_memory_used_bytes", MIB),
+    "memory.free": ("vitals_gpu_memory_free_bytes", MIB),
+    "temperature.gpu": ("vitals_gpu_temp_celsius", 1),
+    "power.draw": ("vitals_gpu_power_watts", 1),
+    "power.limit": ("vitals_gpu_power_limit_watts", 1),
+    "clocks.gr": ("vitals_gpu_freq_mhz", 1),
+    "clocks.mem": ("vitals_gpu_memory_freq_mhz", 1),
+    "fan.speed": ("vitals_gpu_fan_ratio", 0.01),
+}
+REASONS = [
+    "sw_power_cap",
+    "sw_thermal_slowdown",
+    "hw_thermal_slowdown",
+    "hw_slowdown",
+    "hw_power_brake_slowdown",
+]
+REASON_PREFIXES = ["clocks_event_reasons", "clocks_throttle_reasons"]  # newer drivers first
+METRICS = [m for m, _ in NUMBERS.values()] + ["vitals_gpu_slowdown_state"]
+
+
+def slowdown_state(active: dict) -> int:
+    if active.get("hw_slowdown") or active.get("hw_power_brake_slowdown"):
+        return 3
+    if active.get("sw_thermal_slowdown") or active.get("hw_thermal_slowdown"):
+        return 2
+    return 1 if active.get("sw_power_cap") else 0
+
+
+def _rows(text: str) -> list[list[str]]:
+    return [[c.strip() for c in line.split(",")] for line in text.splitlines() if line.strip()]
+
+
+def _number(s: str):
+    try:
+        return float(s)
+    except ValueError:  # "[N/A]", "[Not Supported]"
+        return None
+
+
+def _run(args):
+    p = subprocess.run(args, capture_output=True, text=True, check=False, timeout=10)
+    return p.returncode, p.stdout, p.stderr
+
+
+class NvidiaGpus:
+    def __init__(self, run=_run, smi="nvidia-smi"):
+        self.run, self.smi = run, smi
+        self.reasons_prefix = None  # found on first use; False once neither name works
+
+    def _query(self, fields):
+        code, out, _ = self.run(
+            [self.smi, f"--query-gpu={','.join(fields)}", "--format=csv,noheader,nounits"]
+        )
+        return _rows(out) if code == 0 else None
+
+    def _reasons(self) -> dict:
+        """gpu index -> {reason: active}; {} if this driver reports none."""
+        prefixes = [self.reasons_prefix] if self.reasons_prefix else REASON_PREFIXES
+        if self.reasons_prefix is False:
+            return {}
+        for prefix in prefixes:
+            rows = self._query(["index"] + [f"{prefix}.{r}" for r in REASONS])
+            if rows is not None:
+                self.reasons_prefix = prefix
+                return {row[0]: {r: v == "Active" for r, v in zip(REASONS, row[1:])} for row in rows}
+        self.reasons_prefix = False
+        return {}
+
+    def values(self) -> list[tuple[str, dict, float]]:
+        rows = self._query(FIELDS)
+        if not rows:
+            return []
+        reasons = self._reasons()
+        out = []
+        for row in rows:
+            got = dict(zip(FIELDS, row))
+            labels = {"gpu": got["index"], "name": got["name"]}
+            for field, (metric, scale) in NUMBERS.items():
+                n = _number(got.get(field, ""))
+                if n is not None:
+                    out.append((metric, labels, round(n * scale, 4) if scale != MIB else int(n * scale)))
+            if got["index"] in reasons:
+                out.append(("vitals_gpu_slowdown_state", labels, slowdown_state(reasons[got["index"]])))
+        return out
+
+    def render(self) -> str:
+        return "".join(
+            f"{metric}{label_text({'gpu': lab['gpu'], 'name': lab['name']})} {value}\n"
+            for metric, lab, value in self.values()
+        )
+
+    def snapshot(self) -> str:
+        import json
+
+        return json.dumps([{"metric": m, **lab, "value": v} for m, lab, v in self.values()])
+
+
+def main():
+    from inference_vitals.metrics import run_until_interrupt, serve
+
+    p = argparse.ArgumentParser()
+    p.add_argument("--port", type=int, default=9091)
+    server = serve(NvidiaGpus(smi=shutil.which("nvidia-smi") or "nvidia-smi"), p.parse_args().port)
+    run_until_interrupt(server)
+
+
+if __name__ == "__main__":
+    main()