tuieval 0.2.0.dev62__py3-none-any.whl → 0.2.0.dev66__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev62'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev62')
+__version__ = version = '0.2.0.dev66'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev66')
 
 __commit_id__ = commit_id = None

tuieval/compare.py

@@ -35,7 +35,8 @@ def _load_native(data):
     m, pack, run = data["model"], data["pack"], data["run"]
     info = {"label": m["label"], "served": m.get("served_name"), "tags": m.get("tags") or [],
             "settings": data.get("settings", {}), "load_s": run.get("load_s"),
-            "peak_rss_mb": run.get("peak_rss_mb"), "pack": pack["name"], "pack_fp": pack["fingerprint"],
+            "peak_rss_mb": run.get("peak_rss_mb"), "vram_peak_gb": run.get("vram_peak_gb"),
+            "pack": pack["name"], "pack_fp": pack["fingerprint"],
             # hosted (OpenRouter) models: the provider endpoint and its declared quantization
             "endpoint": run.get("endpoint"), "quantization": run.get("quantization"),
             "loaded_alongside": run.get("loaded_alongside") or []}

tuieval/doctor.py

@@ -10,6 +10,7 @@ import platform
 import sys
 
 from . import engine
+from . import machines
 from . import workspace
 
 OK, WARN, BAD = "✓", "!", "✗"
@@ -44,6 +45,22 @@ def check(models_path=None):
         return r.bad
     r.line(OK, f"workspace {e.root}")
 
+    r.head("Machine")
+    mc = e.machine()
+    r.line(OK, f"{mc.id}: {mc.summary}")
+    if mc.discrete:
+        r.line(OK, f"models are sized to {mc.gpu_limit_gb:.0f} GB of VRAM, keeping {e.headroom_gb(mc):g} GB free "
+                   "(memory_headroom_gb, gpu_memory_gb under [machines.<id>] change it)")
+        new = machines.gpu_id(machines.discrete_gpus(), mc.ram_gb)
+        if mc.id != new and not os.environ.get("EVALS_MACHINE"):
+            r.line(WARN, f"keeping the id {mc.id} this machine had before its GPU was detected, so earlier results "
+                         "and tuning still match", f"to switch: EVALS_MACHINE={new} (results under the old id "
+                                                   "then count as another machine)")
+    elif platform.system() == "Linux" and any(os.path.exists(p) for p in ("/dev/nvidia0", "/dev/kfd")):
+        r.line(WARN, "this machine seems to have an Nvidia or AMD GPU, but nvidia-smi / amd-smi / rocm-smi "
+                     "isn't installed, so models are sized against system RAM",
+               "install the vendor's tools, or set gpu_memory_gb under [machines.<id>] in models.toml")
+
     r.head("Packs")
     for err in e.pack_errors:
         r.line(BAD, err, "fix the pack, then: tuieval selftest")

tuieval/engine.py

@@ -1191,7 +1191,19 @@ class Engine:
 
     # ---- serving on this machine
     def machine(self):
-        return machines.detect()
+        """This machine, with its models.toml [machines.<id>] gpu_memory_gb applied. A Linux machine
+        that recorded itself before its GPU was detected keeps that id, so its results and tuning
+        still match (EVALS_MACHINE renames it)."""
+        mc = machines.detect()
+        if mc.discrete and not os.environ.get("EVALS_MACHINE"):
+            old = machines.machine_id(mc.chip, mc.ram_gb)
+            if os.path.isdir(os.path.join(self.tuning_dir, old)) and \
+                    not os.path.isdir(os.path.join(self.tuning_dir, mc.id)):
+                mc = dataclasses.replace(mc, id=old)
+        override = self.cfg.get("machines", {}).get(mc.id, {}).get("gpu_memory_gb")
+        if override:
+            mc = dataclasses.replace(mc, gpu_limit_gb=float(override))
+        return mc
 
     def machines(self):
         """This machine first, then every other machine recorded in tuning/ (synced)."""
@@ -1203,9 +1215,14 @@ class Engine:
         return [here] + [mc for mc in profiles.known_machines(self.tuning_dir) if mc.id != here.id]
 
     def machine_settings(self, machine_id=None):
-        """models.toml [machines.<id>] (optional): memory_headroom_gb, name."""
+        """models.toml [machines.<id>] (optional): memory_headroom_gb, gpu_memory_gb, name."""
         return self.cfg.get("machines", {}).get(machine_id or self.machine().id, {})
 
+    def headroom_gb(self, mc):
+        """GPU memory kept free: memory_headroom_gb, else 4 GB on a Mac (macOS and other apps share
+        it) and 1.5 GB in a discrete GPU's VRAM."""
+        return float(self.machine_settings(mc.id).get("memory_headroom_gb", 1.5 if mc.discrete else 4.0))
+
     def parallel_models(self):
         """How many models a run serves at a time on this machine: models.toml [machines.<id>]
         parallel_models (default 1)."""
@@ -1224,7 +1241,7 @@ class Engine:
     def memory_available_gb(self):
         """GPU memory models may take here (what the fit check sizes contexts against)."""
         mc = self.machine()
-        return mc.gpu_limit_gb - self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
+        return mc.gpu_limit_gb - self.headroom_gb(mc)
 
     def runs_alone(self, m):
         """A server whose before_start step may stop other servers never runs next to another model."""
@@ -1269,8 +1286,7 @@ class Engine:
                 if fit_check(server):
                     mmproj = expand(m.get("mmproj", ""))
                     extra = os.path.getsize(mmproj) if mmproj and os.path.isfile(mmproj) else 0
-                    headroom = self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
-                    f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=headroom, extra_bytes=extra,
+                    f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=self.headroom_gb(mc), extra_bytes=extra,
                                      want_ctx=want)
                     ctx, fits, note, need = f.max_ctx, f.fits, f.note, f.need_gb
             except (OSError, ValueError) as e:
@@ -1484,11 +1500,22 @@ class Engine:
                         time.sleep(0.2)
         threading.Thread(target=run, daemon=True).start()
 
-    def _watch_memory(self, proc, label, stop, peak):
+    VRAM_EVERY_S = 6   # how often to read a discrete GPU's VRAM while a server runs
+
+    def _watch_memory(self, proc, label, stop, peak, info=None):
         """Peak resident memory of the server's process group (MB). On Apple Silicon this includes
-        the model weights mapped for the GPU."""
+        the model weights mapped for the GPU. With discrete GPUs, also the peak VRAM it holds
+        (info["vram_peak_gb"]): its own processes' on Nvidia, the machine's on AMD."""
+        discrete = info is not None and self.machine().discrete
+        last_vram = [0.0]
+
         def run():
             while not stop.is_set():
+                if discrete and time.time() - last_vram[0] >= self.VRAM_EVERY_S:
+                    last_vram[0] = time.time()
+                    used = machines.vram_used_gb(machines.tree_pids(proc.pid))
+                    if used:
+                        info["vram_peak_gb"] = round(max(info.get("vram_peak_gb") or 0, used), 2)
                 try:
                     out = subprocess.run(["ps", "-axo", "pgid=,rss="], capture_output=True, text=True).stdout
                     rss = sum(int(r) for g, r in (l.split() for l in out.splitlines() if l.strip())
@@ -1789,7 +1816,7 @@ class Engine:
                                           f"(see logs/server/{log_name or m['label']}.log)")
                 t0 = time.time()
                 srv = self._start(cmd, log, cwd=cwd)
-                self._watch_memory(srv, m["label"], stop, peak)
+                self._watch_memory(srv, m["label"], stop, peak, info)
             else:
                 key_env = server.get("api_key_env")
                 if key_env and not os.environ.get(key_env):
@@ -1881,6 +1908,7 @@ class Engine:
             def run_info():
                 return {"load_s": info["load_s"], "peak_rss_mb": info["peak_mb"](),
                         "server_facts": info["facts"] or None, "gpu_peak_gb": info.get("gpu_peak_gb"),
+                        "vram_peak_gb": info.get("vram_peak_gb"),
                         "kernel_share_max": info.get("kernel_share_max"),
                         **({"loaded_alongside": loaded_alongside} if loaded_alongside else {}), **serving_info}
             failed_in_row = 0
@@ -1965,7 +1993,8 @@ class Engine:
         return tests
 
     # ---- sittings: a pack run over several sessions (stopped and resumed) keeps its total time
-    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "kernel_share_max", "machine", "loaded_alongside")
+    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "vram_peak_gb", "kernel_share_max", "machine",
+                    "loaded_alongside")
 
     def _sitting(self, job, info):
         now = time.time()
@@ -2037,7 +2066,7 @@ class Engine:
         out = {"started": min(s["started"] for s in sittings),
                "wall_s": round(sum(s.get("wall_s") or 0 for s in sittings), 1),
                "sittings": len(sittings), "sittings_log": sittings}
-        for k in ("peak_rss_mb", "gpu_peak_gb", "kernel_share_max"):
+        for k in ("peak_rss_mb", "gpu_peak_gb", "vram_peak_gb", "kernel_share_max"):
             if peak(k) is not None:
                 out[k] = peak(k)
         along = sorted({l for s in sittings for l in s.get("loaded_alongside") or []})

tuieval/machines.py

@@ -1,6 +1,7 @@
 """Which machine this is, what a model needs, and whether it fits.
 
-    detect()              -> Machine (chip, cores, RAM, GPU memory limit, id like "m3max-64gb")
+    detect()              -> Machine (chip, cores, RAM, GPU memory limit, id like "m3max-64gb";
+                             with Nvidia/AMD cards on Linux, their VRAM and an id like "rtx4090-128gb")
     read_gguf(path)       -> GGUF header facts (architecture, layers, KV heads, context length, …)
     fit(model_path, …)    -> Fit (largest context that fits this machine, estimated memory)
 
@@ -9,6 +10,7 @@ the KV cache on its own (that would change answers); a quantized-KV variant is a
 """
 import dataclasses
 import functools
+import json
 import os
 import platform
 import re
@@ -28,15 +30,19 @@ class Machine:
     id: str
     chip: str
     ram_gb: float
-    gpu_limit_gb: float      # what Metal may wire for the GPU
+    gpu_limit_gb: float      # what Metal may wire for the GPU; with discrete GPUs, their VRAM (all cards)
     p_cores: int
     e_cores: int
     gpu_cores: int | None
     os: str
+    gpu_name: str | None = None   # discrete GPUs, e.g. "2× NVIDIA GeForce RTX 4090"
+    discrete: bool = False        # the GPU has its own memory (VRAM), separate from RAM
 
     @property
     def summary(self):
         cores = f"{self.p_cores}P+{self.e_cores}E" if self.e_cores else f"{self.p_cores} cores"
+        if self.discrete:
+            return f"{self.chip} · {self.ram_gb:.0f} GB RAM · {self.gpu_name} · {self.gpu_limit_gb:.0f} GB VRAM · {cores}"
         gpu = f", {self.gpu_cores}-core GPU" if self.gpu_cores else ""
         return f"{self.chip} · {self.ram_gb:.0f} GB (GPU up to {self.gpu_limit_gb:.0f} GB) · {cores}{gpu}"
 
@@ -53,6 +59,129 @@ def machine_id(chip, ram_gb):
     return re.sub(r"[^a-z0-9]+", "", name.lower()) + f"-{round(ram_gb)}gb"
 
 
+def gpu_id(gpus, ram_gb):
+    """A machine id from its discrete GPUs: "rtx4090-128gb", "2xrtx4090-128gb", "rx7900xtx+rtx3090-64gb"."""
+    names = []
+    for name in sorted({n for n, _ in gpus}):
+        short = re.sub(r"\b(nvidia|geforce|amd|radeon|ati|graphics|tm)\b", "", name.lower())
+        short = re.sub(r"[^a-z0-9]+", "", short) or "gpu"
+        count = sum(n == name for n, _ in gpus)
+        names.append(f"{count}x{short}" if count > 1 else short)
+    return "+".join(names) + f"-{round(ram_gb)}gb"
+
+
+# ---------------------------------------------------------------- discrete GPUs (Linux: Nvidia, AMD)
+# Read from the vendors' own tools. Anything unexpected counts as "not detected", never an error.
+def _run(cmd, timeout=10):
+    try:
+        p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    return p.stdout if p.returncode == 0 else None
+
+
+def _nvidia_gpus():
+    out = _run(["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"])
+    gpus = []
+    for line in (out or "").splitlines():
+        name, _, mib = line.rpartition(",")
+        try:
+            gpus.append((name.strip(), float(mib) / 1024))
+        except ValueError:
+            pass
+    return [g for g in gpus if g[0] and g[1] > 0]
+
+
+def _walk(obj):
+    """Every dict inside a JSON value."""
+    if isinstance(obj, dict):
+        yield obj
+        for v in obj.values():
+            yield from _walk(v)
+    elif isinstance(obj, list):
+        for v in obj:
+            yield from _walk(v)
+
+
+def _bytes(value, unit=None):
+    """VRAM sizes as amd-smi/rocm-smi print them: 25753026560, "24560 MB", {"value": 24560, "unit": "MB"}."""
+    if isinstance(value, dict):
+        return _bytes(value.get("value"), value.get("unit"))
+    m = re.match(r"\s*([\d.]+)\s*([KMGT]i?B|B)?\s*$", str(value), re.I)
+    if not m:
+        return None
+    scale = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40}
+    u = (m.group(2) or unit or "B").upper().replace("I", "")
+    return float(m.group(1)) * scale.get(u, 1)
+
+
+def _amd_gpus():
+    out = _run(["amd-smi", "static", "--asic", "--vram", "--json"])
+    gpus = []
+    try:
+        for d in _walk(json.loads(out)) if out else []:
+            if "vram" in d and isinstance(d["vram"], dict):
+                size = _bytes(d["vram"].get("size"))
+                name = (d.get("asic") or {}).get("market_name") or "AMD GPU"
+                if size:
+                    gpus.append((str(name), size / GB))
+    except ValueError:
+        gpus = []
+    if gpus:
+        return gpus
+    out = _run(["rocm-smi", "--showmeminfo", "vram", "--showproductname", "--json"])
+    try:
+        cards = json.loads(out) if out else {}
+    except ValueError:
+        return []
+    for key, d in sorted(cards.items()) if isinstance(cards, dict) else []:
+        if not key.startswith("card") or not isinstance(d, dict):
+            continue
+        size = _bytes(d.get("VRAM Total Memory (B)"))
+        name = d.get("Card Series") or d.get("Card series") or d.get("Card SKU") or d.get("Card model") or "AMD GPU"
+        if size:
+            gpus.append((str(name), size / GB))
+    return gpus
+
+
+def discrete_gpus():
+    """[(name, VRAM GB)] for the Nvidia or AMD cards this machine has, or [] (none, or tools missing)."""
+    return _nvidia_gpus() or _amd_gpus()
+
+
+def vram_used_gb(pids=None):
+    """VRAM in use now (GB): by these processes on Nvidia (all of it when pids is None), or on the
+    whole machine on AMD, whose tools don't report it reliably per process. None if unknown."""
+    if pids is not None:
+        out = _run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"])
+        if out is not None:
+            used = 0.0
+            for line in out.splitlines():
+                pid, _, mib = line.partition(",")
+                if pid.strip() in {str(p) for p in pids}:
+                    try:
+                        used += float(mib) / 1024
+                    except ValueError:
+                        pass
+            return used
+    out = _run(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"])
+    if out is not None:
+        try:
+            return sum(float(x) for x in out.split()) / 1024
+        except ValueError:
+            return None
+    out = _run(["rocm-smi", "--showmeminfo", "vram", "--json"])
+    try:
+        cards = json.loads(out) if out else None
+    except ValueError:
+        return None
+    if not isinstance(cards, dict):
+        return None
+    used = [_bytes(d.get("VRAM Total Used Memory (B)")) for k, d in cards.items()
+            if k.startswith("card") and isinstance(d, dict)]
+    return sum(u for u in used if u) / GB if any(used) else None
+
+
 @functools.lru_cache(maxsize=1)
 def detect():
     """This machine. Override the id with EVALS_MACHINE if two machines would collide."""
@@ -73,11 +202,19 @@ def detect():
         except (OSError, subprocess.TimeoutExpired):
             pass
         os_name = "macOS " + platform.mac_ver()[0]
-    else:  # Linux and others: CPU-side facts only
+    else:  # Linux and others: the CPU side, plus Nvidia/AMD cards if their tools are installed
         chip = platform.processor() or platform.machine()
         ram = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / GB
         p, e, gpu_cores, limit = os.cpu_count() or 1, 0, None, ram * 0.9
         os_name = platform.system()
+        gpus = discrete_gpus()
+        if gpus:
+            names = sorted({n for n, _ in gpus})
+            gpu_name = " + ".join(f"{sum(n == x for n, _ in gpus)}× {x}" if sum(n == x for n, _ in gpus) > 1 else x
+                                  for x in names)
+            mid = os.environ.get("EVALS_MACHINE") or gpu_id(gpus, ram)
+            return Machine(mid, chip, round(ram, 1), round(sum(g for _, g in gpus), 1), p, e, None, os_name,
+                           gpu_name=gpu_name, discrete=True)
     mid = os.environ.get("EVALS_MACHINE") or machine_id(chip, ram)
     return Machine(mid, chip, round(ram, 1), round(limit, 1), p, e, gpu_cores, os_name)
 
@@ -195,14 +332,15 @@ def fit(model_path, machine=None, kv_type="f16", headroom_gb=4.0, extra_bytes=0,
     model_max = info["context_length"] or 131072
     max_ctx = int(min(room / per_tok, model_max, want_ctx or model_max)) if room > 0 else 0
     max_ctx = max_ctx // 1024 * 1024
+    of = " of VRAM" if machine.discrete else ""
     if max_ctx < MIN_CTX:
         need = fixed + MIN_CTX * per_tok / GB
         return Fit(False, max_ctx, round(need, 1), round(available, 1),
                    f"doesn't fit on {machine.id}: needs ~{need:.0f} GB at {MIN_CTX // 1024}k context, "
-                   f"{available:.0f} GB available")
+                   f"{available:.0f} GB{of} available")
     need = fixed + max_ctx * per_tok / GB
     return Fit(True, max_ctx, round(need, 1), round(available, 1),
-               f"fits on {machine.id} up to {max_ctx // 1024}k context (~{need:.0f} of {available:.0f} GB)")
+               f"fits on {machine.id} up to {max_ctx // 1024}k context (~{need:.0f} of {available:.0f} GB{of})")
 
 
 # ---------------------------------------------------------------- memory pressure
@@ -287,6 +425,26 @@ def tree_cpu_seconds(pid):
     return (sum(rows[p][1] for p in tree), sum(rows[p][2] for p in tree))
 
 
+def tree_pids(pid):
+    """A process and all its descendants (pids), or [pid] if ps can't be read."""
+    try:
+        out = subprocess.run(["ps", "-axo", "pid=,ppid="], capture_output=True, text=True, timeout=5).stdout
+    except (OSError, subprocess.TimeoutExpired):
+        return [pid]
+    children = {}
+    for line in out.splitlines():
+        parts = line.split()
+        if len(parts) == 2:
+            children.setdefault(parts[1], []).append(parts[0])
+    tree, frontier = {str(pid)}, [str(pid)]
+    while frontier:
+        for child in children.get(frontier.pop(), []):
+            if child not in tree:
+                tree.add(child)
+                frontier.append(child)
+    return sorted(int(p) for p in tree)
+
+
 def group_cpu_seconds(pgid):
     """(user, kernel) CPU seconds used so far by a process group, or None."""
     try:

tuieval/pta.py

@@ -1,4 +1,4 @@
-"""PTA index: parsimony (tokens), time and accuracy per model, a bar each (longer = better).
+"""PTA index: parsimony (tokens), time and accuracy per model, a bar each.
 
     P  parsimony (tokens)  the tokens (reasoning and answer) used on every question compared (each
                            question at its median over repeats), shown as how many times the leanest
@@ -7,8 +7,10 @@
                            model's ("28×").
     A  accuracy            the share of answers to those questions that passed (%).
 
-Behind the P and T bars is a 0-100 score: 100 for the best, POINTS_PER_DOUBLING points less for every
-doubling (0 at about 100 times more), so models far behind the best still get a visible bar.
+The P and T bars are as long as the tokens and time used (the model that used the most fills its bar,
+the others in proportion, at least one block), so shorter is better; the A bar: longer is better.
+P and T also carry a 0-100 score (100 for the best, POINTS_PER_DOUBLING points less for every
+doubling), used to order models with the same accuracy.
 
 Models are compared on the questions all of them answered, so one that skipped some never looks
 faster. Server and connection errors aren't answers. By default a model with fewer than half as many
@@ -93,9 +95,11 @@ def index(rows, models=None, memory=None):
             "most": most}
 
 
-# (letter, name, what the number beside the bar shows, colour); the bar itself is the 0-100 score
-LETTERS = (("P", "parsimony (tokens)", "tokens_x", "cyan"), ("T", "time", "time_x", "magenta"),
-           ("A", "accuracy", "A", "green"))
+# (letter, name, which way is better, what the bar measures, the number beside it, colour). P and T bars
+# are the amount itself (tokens, time: shorter is better); the A bar is the share right (longer is better).
+LETTERS = (("P", "parsimony (tokens)", "▼ shorter is better", "tokens", "tokens_x", "cyan"),
+           ("T", "time", "▼ shorter is better", "total_s", "time_x", "magenta"),
+           ("A", "accuracy", "▲ longer is better", "A", "A", "green"))
 
 
 def times(v):
@@ -103,26 +107,41 @@ def times(v):
     return "-" if v is None else f"{v:.1f}×" if v < 9.95 else f"{v:.0f}×"
 
 
+def _blocks(width, v, most):
+    """How many of width blocks a bar fills: v as a share of most, at least one block when v > 0."""
+    if not v or not most:
+        return 0
+    return max(1, min(width, round(width * v / most)))
+
+
 def bars(result, width=20, markup=True):
-    """Lines with a row per model and a bar per score (P, T, A), longer = better: the at-a-glance view.
-    Beside P and T, how many times the leanest / fastest model's tokens / time; beside A, the percentage.
-    markup=False gives plain text (the report)."""
+    """Lines with a row per model and a bar each for P, T and A: the at-a-glance view. The P and T bars
+    are proportional to tokens and time (the longest is the model that used most; shorter is better),
+    with how many times the leanest / fastest model's beside them; the A bar is the share of answers
+    right (longer is better, full only when every answer passed). markup=False gives plain text."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
-    # each column: the bar, a space, "100%" or "9.8×" (5), then a gap; at least as wide as its heading
-    cell_w = max(width + 9, *(len(f"{k} {name}") + 2 for k, name, _, _ in LETTERS))
-    head = " " * (name_w + 2) + "".join(f"{f'{k} {name}':<{cell_w}}" for k, name, _, _ in LETTERS)
-    lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip()]
+    # each column: the bar, a space, "100%" or "9.8×" (6), then a gap; at least as wide as its heading
+    cell_w = max(width + 9, *(len(t) + 2 for k, name, better, *_ in LETTERS for t in (f"{k} {name}", better)))
+    pad = " " * (name_w + 2)
+    head = pad + "".join(f"{f'{k} {name}':<{cell_w}}" for k, name, *_ in LETTERS)
+    sub = pad + "".join(f"{better:<{cell_w}}" for _, _, better, *_ in LETTERS)
+    lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip(), f"[dim]{sub.rstrip()}[/dim]" if markup else sub.rstrip()]
+    most = {m: max((x.get(m) or 0 for x in result["models"]), default=0) for _, _, _, m, _, _ in LETTERS}
     for x in result["models"]:
         cells = []
-        for k, _, shown, color in LETTERS:
-            v = x[k]
-            full = 0 if v is None else round(width * max(0, min(100, v)) / 100)
-            if v is not None and v < 100:   # full only when perfect (accuracy) or the best (P, T)
-                full = min(full, width - 1)
+        for k, _, _, measure, shown, color in LETTERS:
+            if k == "A":
+                v = x["A"]
+                full = 0 if v is None else round(width * max(0, min(100, v)) / 100)
+                if v is not None and v < 100:   # full only when every answer passed
+                    full = min(full, width - 1)
+                num = "-" if v is None else compare.pct(v / 100)
+            else:
+                full = _blocks(width, x.get(measure), most[measure])
+                num = times(x.get(shown))
             bar, rest = "█" * full, "░" * (width - full)
-            num = (("-" if v is None else compare.pct(v / 100)) if shown == "A" else times(x.get(shown)))
             bar = f"[{color}]{bar}[/][grey37]{rest}[/]" if markup else bar + rest
             cells.append(f"{bar} {num:>6}" + " " * (cell_w - width - 7))
         name = f"{x['model']:<{name_w}}"
@@ -183,8 +202,9 @@ def scope_note(result, models):
     if not result["questions"]:
         return "These models have no question in common yet, so there's nothing to compare."
     return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
-            "Longer bars are better. P parsimony (tokens) and T time: how many times the leanest / fastest "
-            "model's (1.0×, ★ in the table). A accuracy: answers right.")
+            "P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
+            "and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
+            "(longer is better).")
 
 
 def for_engine(e, labels=None, packs=None, results_dir=None):

tuieval/tui.py

@@ -332,9 +332,10 @@ with the model's reasoning and answer as they stream, then recent results with t
   v  with several models at a time: stream the next one (k skips the one streaming)""",
     "ResultsScreen": """[b]Results[/b]
 
-  PTA index              parsimony (tokens), time and accuracy per model, a bar each (longer = better;
-                         2.4× = 2.4 times the best), with per-answer numbers, tok/s, TTFT and memory in
-                         the table. Results open here.
+  PTA index              parsimony (tokens), time and accuracy per model, a bar each: tokens and time
+                         as long as they took (shorter is better; 2.4× = 2.4 times the best), accuracy
+                         longer is better. Per-answer numbers, tok/s, TTFT and memory in the table.
+                         Results open here.
   Production readiness   one verdict per model and use case; the table says why and what to run next
   Per question           every question, model by model (filters: pack, level, Models disagree)
   Failures               every wrong answer with the grader's reason; enter opens it in full

tuieval/tune.py

@@ -60,6 +60,7 @@ SWAP_NOTABLE = 256 * 2**20  # swap worth a warning (bytes); rejecting for swap n
 ANSWER_TOKENS = 1024       # answer length the projected objective scores ([tune] answer_tokens)
 PRESSURE_CRITICAL = 4      # macOS memory pressure level that rejects a candidate
 PRESSURE_EVERY_S = 2
+VRAM_MARGIN_GB = 0.25     # a discrete GPU this close to full could run out on a longer prompt
 GPU_MARGIN_GB = 0.75      # a candidate whose GPU allocation comes this close to the residency limit
                           # is rejected even before it stalls: servers grow as requests arrive
 REQUEST_LIMIT_S = 600     # a tuning request taking longer fails the candidate ([tune] request_timeout_s)
@@ -556,6 +557,14 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                     r.facts["gpu_settled_gb"] = round(settled, 1)
                 if info.get("kernel_share_max") is not None:
                     r.facts["kernel_share"] = info["kernel_share_max"]
+                mc = eng.machine()
+                if mc.discrete:   # VRAM is the limit there: its own memory, nothing else can use it
+                    vram = machines.vram_used_gb()
+                    if vram is not None:
+                        r.facts["vram_used_gb"] = round(vram, 1)
+                        if vram > mc.gpu_limit_gb - VRAM_MARGIN_GB and not r.error:
+                            r.error = (f"VRAM nearly full: {vram:.1f} of {mc.gpu_limit_gb:.0f} GB in use "
+                                       "(a longer prompt could run out)")
                 limit = eng.gpu_residency_gb()
                 if settled and settled > limit - GPU_MARGIN_GB and not r.error:
                     r.error = (f"GPU allocation settled at {settled:.1f} GB, within {GPU_MARGIN_GB} GB of the "

{tuieval-0.2.0.dev62.dist-info → tuieval-0.2.0.dev66.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev62
+Version: 0.2.0.dev66
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -51,7 +51,7 @@ Whatever runs your models, the first steps are short:
 | You have | Do this |
 |---|---|
 | **LM Studio, Ollama or vLLM** | Start its server and load a model, then `tuieval init my-evals`: it finds the model. Later: `tuieval add` lists new ones. |
-| **GGUF files** | Install llama.cpp (`brew install llama.cpp`), then `tuieval add ~/models/Some-Model-Q4_K_M.gguf`. tuieval starts `llama-server` for it, sized to your machine. |
+| **GGUF files** | Install llama.cpp (`brew install llama.cpp`, or a CUDA/ROCm build on Linux), then `tuieval add ~/models/Some-Model-Q4_K_M.gguf`. tuieval starts `llama-server` for it, sized to your Mac's memory or your Nvidia/AMD card's VRAM. |
 | **An OpenRouter key** | `export OPENROUTER_API_KEY=…`, then `tuieval run --tier smoke --only openrouter:<model id>`. |
 
 Stuck? `tuieval doctor` checks servers, models, packs and keys, and says what to fix.
@@ -76,7 +76,7 @@ Afterwards, every answer keeps its reasoning. Enter on a result opens it with th
 - **Verdicts you can act on.** A pack passes only with zero critical failures over enough trials, and an accuracy whose 95% lower bound clears your bar. *INCONCLUSIVE* says what evidence is missing.
 - **Two tiers:** *Screen* a sample of every pack to drop weak models fast, then *Certify* finalists on every test with repeats. Certification reuses the screening answers.
 - **Honest numbers:** every request is a fresh single-turn conversation, with prompt caching off and repeat rounds in different orders. Each answer records what was sent, and the run warns about reused prompts or identical repeats.
-- **Per-machine speed.** A fit check picks the largest context that fits your Mac's GPU memory, `tuieval tune` finds the fastest speed-only server flags (with a guard that rejects flags that change answers), and readiness includes a *Fast enough?* table per machine, measured or projected.
+- **Per-machine speed.** A fit check picks the largest context that fits your Mac's GPU memory or your Nvidia/AMD card's VRAM, `tuieval tune` finds the fastest speed-only server flags (with a guard that rejects flags that change answers), and readiness includes a *Fast enough?* table per machine, measured or projected.
 - **Tests that check themselves.** Every test carries a reference answer and known-wrong answers, and `tuieval selftest` checks the grader accepts the first and rejects the second, and that each gate is reachable at all.
 - **History.** Verdict changes are appended to `results/verdicts.jsonl`, and replaced results are moved to `history/`, never overwritten.
 

{tuieval-0.2.0.dev62.dist-info → tuieval-0.2.0.dev66.dist-info}/RECORD

@@ -1,24 +1,24 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=Q5t2lKtGGZuhOAPSq1URLDDN-gL2EG-f55AiFxUzLv0,535
+tuieval/_version.py,sha256=3oGXG-ulb5H7C2aLJwOF-Gqqfw6csmjHt4syzml7C4M,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=fn6z0cupM-aW3fY5fFgnS-HmOQmMzlDg-U76Y6LgL3o,4510
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
-tuieval/compare.py,sha256=V_K34LFnDvd7xhGDS4zZDCguaIMU05f5mCbPx0VpUOA,26944
-tuieval/doctor.py,sha256=y1C_ggjd8oikXTrovgTBf26wjThP8EoVkfTVeO_n5dQ,6070
-tuieval/engine.py,sha256=0PuHFus1k02qAY8nlsI6z5iQo7IT82bMwmHDZT6fGtU,114784
+tuieval/compare.py,sha256=T6a_PC7z-jHEvw09KEceo2dxoLMi-9cHYPQs4L6Ctig,26997
+tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
+tuieval/engine.py,sha256=SsXqR1ydn827qBX8NoawyT2roNuelmrZB-hOuyGSesY,116514
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
-tuieval/machines.py,sha256=tdRrn3WNOmCHwfcxcWz1NuAuOJdI7WnJFR1y6xgF89c,13512
+tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
-tuieval/pta.py,sha256=6HwPMvTTJFoZjO-2ZurucJrp1itIzpDTKL1xvYP9W0w,10888
+tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=1zNkCdFt744cqpebqiyj-sZrbOI22c6IYBcXk9koz20,46538
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=lrIv4z21s9OG809706Z-lE2-TZ6gg8z0AjaJ_zW7zkw,161021
-tuieval/tune.py,sha256=jXwYuSjfdqbLCE3WonP2xaUHwk_FnhI67twB1BaHggI,38992
+tuieval/tui.py,sha256=LPnUYbynZiHUBv_d7TNttd7smoTlphPHEa5yoGCvU6s,161108
+tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=VB-0LTxR9y8bJHT9RicxtE9K7cUHBdCu40hOCrsbmA4,19150
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
@@ -46,8 +46,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev62.dist-info/METADATA,sha256=X2v6zWEn3E0Lkm-W2x-pjAHgG6qjjzFw_BqeSvyyIxI,13898
-tuieval-0.2.0.dev62.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev62.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev62.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev62.dist-info/RECORD,,
+tuieval-0.2.0.dev66.dist-info/METADATA,sha256=3R7yYobJYnLLsMP_OHfy74PYErGtGBOf50xMtXpZSXA,13996
+tuieval-0.2.0.dev66.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev66.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev66.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev66.dist-info/RECORD,,