tuieval 0.2.0.dev56__py3-none-any.whl → 0.2.0.dev65__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev56'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev56')
+__version__ = version = '0.2.0.dev65'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev65')
 
 __commit_id__ = commit_id = None

tuieval/cli.py

@@ -14,7 +14,7 @@ Start here
 
 Day to day
   tuieval compare [...]       scorecards (--speed, --pairwise, --failures)
-  tuieval pta                 the PTA index: parsimony, speed and accuracy per model, as bars
+  tuieval pta                 the PTA index: parsimony (tokens), time and accuracy per model, as bars
   tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
   tuieval items               tests that don't separate models or look broken

tuieval/compare.py

@@ -35,7 +35,8 @@ def _load_native(data):
     m, pack, run = data["model"], data["pack"], data["run"]
     info = {"label": m["label"], "served": m.get("served_name"), "tags": m.get("tags") or [],
             "settings": data.get("settings", {}), "load_s": run.get("load_s"),
-            "peak_rss_mb": run.get("peak_rss_mb"), "pack": pack["name"], "pack_fp": pack["fingerprint"],
+            "peak_rss_mb": run.get("peak_rss_mb"), "vram_peak_gb": run.get("vram_peak_gb"),
+            "pack": pack["name"], "pack_fp": pack["fingerprint"],
             # hosted (OpenRouter) models: the provider endpoint and its declared quantization
             "endpoint": run.get("endpoint"), "quantization": run.get("quantization"),
             "loaded_alongside": run.get("loaded_alongside") or []}
@@ -121,6 +122,12 @@ def p90(values):
     return statistics.quantiles(values, n=10)[-1] if len(values) > 1 else (values[0] if values else None)
 
 
+def pct(fraction):
+    """A share as a percentage: "100%" and "0%" only when exact, else one decimal ("99.6%"), so a model
+    with a wrong answer never shows as perfect."""
+    return "100%" if fraction >= 1 else "0%" if fraction <= 0 else f"{100 * fraction:.1f}%"
+
+
 def median(values):
     values = [v for v in values if v is not None]
     return statistics.median(values) if values else None
@@ -154,10 +161,19 @@ def settings_notes(infos):
         per_model.setdefault(i["label"], i["settings"])
     keys = sorted({k for s in per_model.values() for k in s})
     differing = [k for k in keys if len({json.dumps(s.get(k)) for s in per_model.values()}) > 1]
-    if differing:
-        detail = "; ".join(f"{k}: " + ", ".join(f"{m}={per_model[m].get(k)}" for m in sorted(per_model))
-                           for k in differing)
-        notes.append(f"Settings differ between models ({detail}). Compare those models knowing that.")
+    parts = []
+    for k in differing:   # only the models that differ from the most common value, named when few
+        groups = collections.defaultdict(list)
+        for m in sorted(per_model):
+            groups[json.dumps(per_model[m].get(k))].append(m)
+        common = max(groups, key=lambda v: len(groups[v]))
+        for value, ms in sorted(groups.items()):
+            if value != common:
+                who = ", ".join(ms) if len(ms) <= 3 else f"{len(ms)} models"
+                shown = "unset" if value == "null" else json.loads(value)
+                parts.append(f"{k} {shown} on {who}")
+    if parts:
+        notes.append("Settings differ: " + "; ".join(parts) + ".")
     fps = collections.defaultdict(set)
     for i in infos:
         if i.get("pack"):
@@ -412,7 +428,7 @@ def by_difficulty(rows):
             cells = []
             for lvl in LEVELS:
                 x = [r["ok"] for r in sub if r.get("difficulty") == lvl]
-                cells.append(f"{sum(x)}/{len(x)} ({100 * sum(x) / len(x):.0f}%)" if x else "-")
+                cells.append(f"{sum(x)}/{len(x)} ({pct(sum(x) / len(x))})" if x else "-")
             table.append([m, s] + cells)
     return header, table
 

tuieval/doctor.py

@@ -10,6 +10,7 @@ import platform
 import sys
 
 from . import engine
+from . import machines
 from . import workspace
 
 OK, WARN, BAD = "✓", "!", "✗"
@@ -44,6 +45,22 @@ def check(models_path=None):
         return r.bad
     r.line(OK, f"workspace {e.root}")
 
+    r.head("Machine")
+    mc = e.machine()
+    r.line(OK, f"{mc.id}: {mc.summary}")
+    if mc.discrete:
+        r.line(OK, f"models are sized to {mc.gpu_limit_gb:.0f} GB of VRAM, keeping {e.headroom_gb(mc):g} GB free "
+                   "(memory_headroom_gb, gpu_memory_gb under [machines.<id>] change it)")
+        new = machines.gpu_id(machines.discrete_gpus(), mc.ram_gb)
+        if mc.id != new and not os.environ.get("EVALS_MACHINE"):
+            r.line(WARN, f"keeping the id {mc.id} this machine had before its GPU was detected, so earlier results "
+                         "and tuning still match", f"to switch: EVALS_MACHINE={new} (results under the old id "
+                                                   "then count as another machine)")
+    elif platform.system() == "Linux" and any(os.path.exists(p) for p in ("/dev/nvidia0", "/dev/kfd")):
+        r.line(WARN, "this machine seems to have an Nvidia or AMD GPU, but nvidia-smi / amd-smi / rocm-smi "
+                     "isn't installed, so models are sized against system RAM",
+               "install the vendor's tools, or set gpu_memory_gb under [machines.<id>] in models.toml")
+
     r.head("Packs")
     for err in e.pack_errors:
         r.line(BAD, err, "fix the pack, then: tuieval selftest")

tuieval/engine.py

@@ -1191,7 +1191,19 @@ class Engine:
 
     # ---- serving on this machine
     def machine(self):
-        return machines.detect()
+        """This machine, with its models.toml [machines.<id>] gpu_memory_gb applied. A Linux machine
+        that recorded itself before its GPU was detected keeps that id, so its results and tuning
+        still match (EVALS_MACHINE renames it)."""
+        mc = machines.detect()
+        if mc.discrete and not os.environ.get("EVALS_MACHINE"):
+            old = machines.machine_id(mc.chip, mc.ram_gb)
+            if os.path.isdir(os.path.join(self.tuning_dir, old)) and \
+                    not os.path.isdir(os.path.join(self.tuning_dir, mc.id)):
+                mc = dataclasses.replace(mc, id=old)
+        override = self.cfg.get("machines", {}).get(mc.id, {}).get("gpu_memory_gb")
+        if override:
+            mc = dataclasses.replace(mc, gpu_limit_gb=float(override))
+        return mc
 
     def machines(self):
         """This machine first, then every other machine recorded in tuning/ (synced)."""
@@ -1203,9 +1215,14 @@ class Engine:
         return [here] + [mc for mc in profiles.known_machines(self.tuning_dir) if mc.id != here.id]
 
     def machine_settings(self, machine_id=None):
-        """models.toml [machines.<id>] (optional): memory_headroom_gb, name."""
+        """models.toml [machines.<id>] (optional): memory_headroom_gb, gpu_memory_gb, name."""
         return self.cfg.get("machines", {}).get(machine_id or self.machine().id, {})
 
+    def headroom_gb(self, mc):
+        """GPU memory kept free: memory_headroom_gb, else 4 GB on a Mac (macOS and other apps share
+        it) and 1.5 GB in a discrete GPU's VRAM."""
+        return float(self.machine_settings(mc.id).get("memory_headroom_gb", 1.5 if mc.discrete else 4.0))
+
     def parallel_models(self):
         """How many models a run serves at a time on this machine: models.toml [machines.<id>]
         parallel_models (default 1)."""
@@ -1224,7 +1241,7 @@ class Engine:
     def memory_available_gb(self):
         """GPU memory models may take here (what the fit check sizes contexts against)."""
         mc = self.machine()
-        return mc.gpu_limit_gb - self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
+        return mc.gpu_limit_gb - self.headroom_gb(mc)
 
     def runs_alone(self, m):
         """A server whose before_start step may stop other servers never runs next to another model."""
@@ -1269,8 +1286,7 @@ class Engine:
                 if fit_check(server):
                     mmproj = expand(m.get("mmproj", ""))
                     extra = os.path.getsize(mmproj) if mmproj and os.path.isfile(mmproj) else 0
-                    headroom = self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
-                    f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=headroom, extra_bytes=extra,
+                    f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=self.headroom_gb(mc), extra_bytes=extra,
                                      want_ctx=want)
                     ctx, fits, note, need = f.max_ctx, f.fits, f.note, f.need_gb
             except (OSError, ValueError) as e:
@@ -1484,11 +1500,22 @@ class Engine:
                         time.sleep(0.2)
         threading.Thread(target=run, daemon=True).start()
 
-    def _watch_memory(self, proc, label, stop, peak):
+    VRAM_EVERY_S = 6   # how often to read a discrete GPU's VRAM while a server runs
+
+    def _watch_memory(self, proc, label, stop, peak, info=None):
         """Peak resident memory of the server's process group (MB). On Apple Silicon this includes
-        the model weights mapped for the GPU."""
+        the model weights mapped for the GPU. With discrete GPUs, also the peak VRAM it holds
+        (info["vram_peak_gb"]): its own processes' on Nvidia, the machine's on AMD."""
+        discrete = info is not None and self.machine().discrete
+        last_vram = [0.0]
+
         def run():
             while not stop.is_set():
+                if discrete and time.time() - last_vram[0] >= self.VRAM_EVERY_S:
+                    last_vram[0] = time.time()
+                    used = machines.vram_used_gb(machines.tree_pids(proc.pid))
+                    if used:
+                        info["vram_peak_gb"] = round(max(info.get("vram_peak_gb") or 0, used), 2)
                 try:
                     out = subprocess.run(["ps", "-axo", "pgid=,rss="], capture_output=True, text=True).stdout
                     rss = sum(int(r) for g, r in (l.split() for l in out.splitlines() if l.strip())
@@ -1789,7 +1816,7 @@ class Engine:
                                           f"(see logs/server/{log_name or m['label']}.log)")
                 t0 = time.time()
                 srv = self._start(cmd, log, cwd=cwd)
-                self._watch_memory(srv, m["label"], stop, peak)
+                self._watch_memory(srv, m["label"], stop, peak, info)
             else:
                 key_env = server.get("api_key_env")
                 if key_env and not os.environ.get(key_env):
@@ -1881,6 +1908,7 @@ class Engine:
             def run_info():
                 return {"load_s": info["load_s"], "peak_rss_mb": info["peak_mb"](),
                         "server_facts": info["facts"] or None, "gpu_peak_gb": info.get("gpu_peak_gb"),
+                        "vram_peak_gb": info.get("vram_peak_gb"),
                         "kernel_share_max": info.get("kernel_share_max"),
                         **({"loaded_alongside": loaded_alongside} if loaded_alongside else {}), **serving_info}
             failed_in_row = 0
@@ -1965,7 +1993,8 @@ class Engine:
         return tests
 
     # ---- sittings: a pack run over several sessions (stopped and resumed) keeps its total time
-    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "kernel_share_max", "machine", "loaded_alongside")
+    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "vram_peak_gb", "kernel_share_max", "machine",
+                    "loaded_alongside")
 
     def _sitting(self, job, info):
         now = time.time()
@@ -2037,7 +2066,7 @@ class Engine:
         out = {"started": min(s["started"] for s in sittings),
                "wall_s": round(sum(s.get("wall_s") or 0 for s in sittings), 1),
                "sittings": len(sittings), "sittings_log": sittings}
-        for k in ("peak_rss_mb", "gpu_peak_gb", "kernel_share_max"):
+        for k in ("peak_rss_mb", "gpu_peak_gb", "vram_peak_gb", "kernel_share_max"):
             if peak(k) is not None:
                 out[k] = peak(k)
         along = sorted({l for s in sittings for l in s.get("loaded_alongside") or []})

tuieval/machines.py

@@ -1,6 +1,7 @@
 """Which machine this is, what a model needs, and whether it fits.
 
-    detect()              -> Machine (chip, cores, RAM, GPU memory limit, id like "m3max-64gb")
+    detect()              -> Machine (chip, cores, RAM, GPU memory limit, id like "m3max-64gb";
+                             with Nvidia/AMD cards on Linux, their VRAM and an id like "rtx4090-128gb")
     read_gguf(path)       -> GGUF header facts (architecture, layers, KV heads, context length, …)
     fit(model_path, …)    -> Fit (largest context that fits this machine, estimated memory)
 
@@ -9,6 +10,7 @@ the KV cache on its own (that would change answers); a quantized-KV variant is a
 """
 import dataclasses
 import functools
+import json
 import os
 import platform
 import re
@@ -28,15 +30,19 @@ class Machine:
     id: str
     chip: str
     ram_gb: float
-    gpu_limit_gb: float      # what Metal may wire for the GPU
+    gpu_limit_gb: float      # what Metal may wire for the GPU; with discrete GPUs, their VRAM (all cards)
     p_cores: int
     e_cores: int
     gpu_cores: int | None
     os: str
+    gpu_name: str | None = None   # discrete GPUs, e.g. "2× NVIDIA GeForce RTX 4090"
+    discrete: bool = False        # the GPU has its own memory (VRAM), separate from RAM
 
     @property
     def summary(self):
         cores = f"{self.p_cores}P+{self.e_cores}E" if self.e_cores else f"{self.p_cores} cores"
+        if self.discrete:
+            return f"{self.chip} · {self.ram_gb:.0f} GB RAM · {self.gpu_name} · {self.gpu_limit_gb:.0f} GB VRAM · {cores}"
         gpu = f", {self.gpu_cores}-core GPU" if self.gpu_cores else ""
         return f"{self.chip} · {self.ram_gb:.0f} GB (GPU up to {self.gpu_limit_gb:.0f} GB) · {cores}{gpu}"
 
@@ -53,6 +59,129 @@ def machine_id(chip, ram_gb):
     return re.sub(r"[^a-z0-9]+", "", name.lower()) + f"-{round(ram_gb)}gb"
 
 
+def gpu_id(gpus, ram_gb):
+    """A machine id from its discrete GPUs: "rtx4090-128gb", "2xrtx4090-128gb", "rx7900xtx+rtx3090-64gb"."""
+    names = []
+    for name in sorted({n for n, _ in gpus}):
+        short = re.sub(r"\b(nvidia|geforce|amd|radeon|ati|graphics|tm)\b", "", name.lower())
+        short = re.sub(r"[^a-z0-9]+", "", short) or "gpu"
+        count = sum(n == name for n, _ in gpus)
+        names.append(f"{count}x{short}" if count > 1 else short)
+    return "+".join(names) + f"-{round(ram_gb)}gb"
+
+
+# ---------------------------------------------------------------- discrete GPUs (Linux: Nvidia, AMD)
+# Read from the vendors' own tools. Anything unexpected counts as "not detected", never an error.
+def _run(cmd, timeout=10):
+    try:
+        p = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+    return p.stdout if p.returncode == 0 else None
+
+
+def _nvidia_gpus():
+    out = _run(["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"])
+    gpus = []
+    for line in (out or "").splitlines():
+        name, _, mib = line.rpartition(",")
+        try:
+            gpus.append((name.strip(), float(mib) / 1024))
+        except ValueError:
+            pass
+    return [g for g in gpus if g[0] and g[1] > 0]
+
+
+def _walk(obj):
+    """Every dict inside a JSON value."""
+    if isinstance(obj, dict):
+        yield obj
+        for v in obj.values():
+            yield from _walk(v)
+    elif isinstance(obj, list):
+        for v in obj:
+            yield from _walk(v)
+
+
+def _bytes(value, unit=None):
+    """VRAM sizes as amd-smi/rocm-smi print them: 25753026560, "24560 MB", {"value": 24560, "unit": "MB"}."""
+    if isinstance(value, dict):
+        return _bytes(value.get("value"), value.get("unit"))
+    m = re.match(r"\s*([\d.]+)\s*([KMGT]i?B|B)?\s*$", str(value), re.I)
+    if not m:
+        return None
+    scale = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40}
+    u = (m.group(2) or unit or "B").upper().replace("I", "")
+    return float(m.group(1)) * scale.get(u, 1)
+
+
+def _amd_gpus():
+    out = _run(["amd-smi", "static", "--asic", "--vram", "--json"])
+    gpus = []
+    try:
+        for d in _walk(json.loads(out)) if out else []:
+            if "vram" in d and isinstance(d["vram"], dict):
+                size = _bytes(d["vram"].get("size"))
+                name = (d.get("asic") or {}).get("market_name") or "AMD GPU"
+                if size:
+                    gpus.append((str(name), size / GB))
+    except ValueError:
+        gpus = []
+    if gpus:
+        return gpus
+    out = _run(["rocm-smi", "--showmeminfo", "vram", "--showproductname", "--json"])
+    try:
+        cards = json.loads(out) if out else {}
+    except ValueError:
+        return []
+    for key, d in sorted(cards.items()) if isinstance(cards, dict) else []:
+        if not key.startswith("card") or not isinstance(d, dict):
+            continue
+        size = _bytes(d.get("VRAM Total Memory (B)"))
+        name = d.get("Card Series") or d.get("Card series") or d.get("Card SKU") or d.get("Card model") or "AMD GPU"
+        if size:
+            gpus.append((str(name), size / GB))
+    return gpus
+
+
+def discrete_gpus():
+    """[(name, VRAM GB)] for the Nvidia or AMD cards this machine has, or [] (none, or tools missing)."""
+    return _nvidia_gpus() or _amd_gpus()
+
+
+def vram_used_gb(pids=None):
+    """VRAM in use now (GB): by these processes on Nvidia (all of it when pids is None), or on the
+    whole machine on AMD, whose tools don't report it reliably per process. None if unknown."""
+    if pids is not None:
+        out = _run(["nvidia-smi", "--query-compute-apps=pid,used_memory", "--format=csv,noheader,nounits"])
+        if out is not None:
+            used = 0.0
+            for line in out.splitlines():
+                pid, _, mib = line.partition(",")
+                if pid.strip() in {str(p) for p in pids}:
+                    try:
+                        used += float(mib) / 1024
+                    except ValueError:
+                        pass
+            return used
+    out = _run(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits"])
+    if out is not None:
+        try:
+            return sum(float(x) for x in out.split()) / 1024
+        except ValueError:
+            return None
+    out = _run(["rocm-smi", "--showmeminfo", "vram", "--json"])
+    try:
+        cards = json.loads(out) if out else None
+    except ValueError:
+        return None
+    if not isinstance(cards, dict):
+        return None
+    used = [_bytes(d.get("VRAM Total Used Memory (B)")) for k, d in cards.items()
+            if k.startswith("card") and isinstance(d, dict)]
+    return sum(u for u in used if u) / GB if any(used) else None
+
+
 @functools.lru_cache(maxsize=1)
 def detect():
     """This machine. Override the id with EVALS_MACHINE if two machines would collide."""
@@ -73,11 +202,19 @@ def detect():
         except (OSError, subprocess.TimeoutExpired):
             pass
         os_name = "macOS " + platform.mac_ver()[0]
-    else:  # Linux and others: CPU-side facts only
+    else:  # Linux and others: the CPU side, plus Nvidia/AMD cards if their tools are installed
         chip = platform.processor() or platform.machine()
         ram = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / GB
         p, e, gpu_cores, limit = os.cpu_count() or 1, 0, None, ram * 0.9
         os_name = platform.system()
+        gpus = discrete_gpus()
+        if gpus:
+            names = sorted({n for n, _ in gpus})
+            gpu_name = " + ".join(f"{sum(n == x for n, _ in gpus)}× {x}" if sum(n == x for n, _ in gpus) > 1 else x
+                                  for x in names)
+            mid = os.environ.get("EVALS_MACHINE") or gpu_id(gpus, ram)
+            return Machine(mid, chip, round(ram, 1), round(sum(g for _, g in gpus), 1), p, e, None, os_name,
+                           gpu_name=gpu_name, discrete=True)
     mid = os.environ.get("EVALS_MACHINE") or machine_id(chip, ram)
     return Machine(mid, chip, round(ram, 1), round(limit, 1), p, e, gpu_cores, os_name)
 
@@ -195,14 +332,15 @@ def fit(model_path, machine=None, kv_type="f16", headroom_gb=4.0, extra_bytes=0,
     model_max = info["context_length"] or 131072
     max_ctx = int(min(room / per_tok, model_max, want_ctx or model_max)) if room > 0 else 0
     max_ctx = max_ctx // 1024 * 1024
+    of = " of VRAM" if machine.discrete else ""
     if max_ctx < MIN_CTX:
         need = fixed + MIN_CTX * per_tok / GB
         return Fit(False, max_ctx, round(need, 1), round(available, 1),
                    f"doesn't fit on {machine.id}: needs ~{need:.0f} GB at {MIN_CTX // 1024}k context, "
-                   f"{available:.0f} GB available")
+                   f"{available:.0f} GB{of} available")
     need = fixed + max_ctx * per_tok / GB
     return Fit(True, max_ctx, round(need, 1), round(available, 1),
-               f"fits on {machine.id} up to {max_ctx // 1024}k context (~{need:.0f} of {available:.0f} GB)")
+               f"fits on {machine.id} up to {max_ctx // 1024}k context (~{need:.0f} of {available:.0f} GB{of})")
 
 
 # ---------------------------------------------------------------- memory pressure
@@ -287,6 +425,26 @@ def tree_cpu_seconds(pid):
     return (sum(rows[p][1] for p in tree), sum(rows[p][2] for p in tree))
 
 
+def tree_pids(pid):
+    """A process and all its descendants (pids), or [pid] if ps can't be read."""
+    try:
+        out = subprocess.run(["ps", "-axo", "pid=,ppid="], capture_output=True, text=True, timeout=5).stdout
+    except (OSError, subprocess.TimeoutExpired):
+        return [pid]
+    children = {}
+    for line in out.splitlines():
+        parts = line.split()
+        if len(parts) == 2:
+            children.setdefault(parts[1], []).append(parts[0])
+    tree, frontier = {str(pid)}, [str(pid)]
+    while frontier:
+        for child in children.get(frontier.pop(), []):
+            if child not in tree:
+                tree.add(child)
+                frontier.append(child)
+    return sorted(int(p) for p in tree)
+
+
 def group_cpu_seconds(pgid):
     """(user, kernel) CPU seconds used so far by a process group, or None."""
     try:

tuieval/pta.py

@@ -1,13 +1,14 @@
-"""PTA index: parsimony, speed (time) and accuracy per model, 0-100 each and higher is better, shown
-as a bar per score for each model.
+"""PTA index: parsimony (tokens), time and accuracy per model, a bar each (longer = better).
 
-    P  parsimony  a score out of 100, from the tokens (reasoning and answer) used on every question
-                  compared (each question at its median over repeats): 100 for the model that used the
-                  fewest, POINTS_PER_DOUBLING points less for every doubling (0 at about 100 times more).
-                  Unlike T it doesn't depend on the machine: it's how much a model says to get there.
-    T  speed      a score out of 100, the same from the total time to answer those questions: 100 for
-                  the fastest model. Log scales, so models that are all far behind the best still differ.
-    A  accuracy   a percentage: the share of answers to those questions that passed.
+    P  parsimony (tokens)  the tokens (reasoning and answer) used on every question compared (each
+                           question at its median over repeats), shown as how many times the leanest
+                           model's ("2.4×"). Unlike T it doesn't depend on the machine.
+    T  time                the total time to answer those questions, as how many times the fastest
+                           model's ("28×").
+    A  accuracy            the share of answers to those questions that passed (%).
+
+Behind the P and T bars is a 0-100 score: 100 for the best, POINTS_PER_DOUBLING points less for every
+doubling (0 at about 100 times more), so models far behind the best still get a visible bar.
 
 Models are compared on the questions all of them answered, so one that skipped some never looks
 faster. Server and connection errors aren't answers. By default a model with fewer than half as many
@@ -32,12 +33,23 @@ def score(total, best):
     return max(0.0, 100 - POINTS_PER_DOUBLING * math.log2(total / best))
 
 
-def index(rows, models=None):
+def memory_gb(infos):
+    """{model: the most memory its server held in any run (GB)}, for models tuieval served (hosted: none).
+    On a discrete GPU that's its peak VRAM (vram_peak_gb), where the model lives; otherwise its resident memory."""
+    out = {}
+    for i in infos:
+        gb = i.get("vram_peak_gb") or (i["peak_rss_mb"] / 1024 if i.get("peak_rss_mb") else None)
+        if gb:
+            out[i["label"]] = max(out.get(i["label"], 0), gb)
+    return out
+
+
+def index(rows, models=None, memory=None):
     """{"questions": n shared, "models": [{model, P, T, A, tokens, total_s, passed, answers, alongside,
-    fewest, fastest}],
+    fewest, fastest, tps, ttft, memory_gb}],
     "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
     most-answered model}, best accuracy first. models: the ones to compare (default: every model in rows
-    with enough answers; models named here are compared whatever their count)."""
+    with enough answers; models named here are compared whatever their count). memory: memory_gb()."""
     no_answers = sorted({r["model"] for r in rows} - {r["model"] for r in rows if not r.get("error")})
     rows = [r for r in rows if not r.get("error")]     # a server or connection error isn't an answer
     tests = compare.per_question(rows)
@@ -61,12 +73,19 @@ def index(rows, models=None):
                     "total_s": sum(c["secs"] or 0 for c in cells),
                     "passed": sum(c["passed"] for c in cells), "answers": answers,
                     "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
+                    # generation speed and time to first token over the same answers (medians)
+                    "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
+                    "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
+                    "memory_gb": (memory or {}).get(m),
                     # answers that ran while other models were served shared the machine's speed
                     "alongside": any(r.get("alongside") for c in cells for r in c["rows"])})
     fastest = min((x["total_s"] for x in out if x["total_s"] > 0), default=None)
     fewest = min((x["tokens"] for x in out if x["tokens"] > 0), default=None)
     for x in out:
         x["P"], x["T"] = score(x["tokens"], fewest), score(x["total_s"], fastest)
+        # shown on the bars: how many times the leanest / fastest model's total (1.0 = the best)
+        x["tokens_x"] = x["tokens"] / fewest if fewest and x["tokens"] else None
+        x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
         x["fewest"] = bool(fewest) and x["tokens"] == fewest
         x["fastest"] = bool(fastest) and x["total_s"] == fastest
     out.sort(key=lambda x: (-(x["A"] or 0), -(x["T"] or 0), x["model"]))
@@ -74,29 +93,40 @@ def index(rows, models=None):
             "most": most}
 
 
-LETTERS = (("P", "parsimony", "/100", "cyan"), ("T", "speed", "/100", "magenta"), ("A", "accuracy", "%", "green"))
+# (letter, name, what the number beside the bar shows, colour); the bar itself is the 0-100 score
+LETTERS = (("P", "parsimony (tokens)", "tokens_x", "cyan"), ("T", "time", "time_x", "magenta"),
+           ("A", "accuracy", "A", "green"))
+
+
+def times(v):
+    """A multiple of the best: "1.0×", "2.4×", "28×"."""
+    return "-" if v is None else f"{v:.1f}×" if v < 9.95 else f"{v:.0f}×"
 
 
 def bars(result, width=20, markup=True):
-    """Lines with a row per model and a bar per score (P, T, A), longer = better, each with its number:
-    the at-a-glance view. markup=False gives plain text (the report)."""
+    """Lines with a row per model and a bar per score (P, T, A), longer = better: the at-a-glance view.
+    Beside P and T, how many times the leanest / fastest model's tokens / time; beside A, the percentage.
+    markup=False gives plain text (the report)."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
-    cell_w = width + 6                          # the bar, a space and "100%" or " 100"
+    # each column: the bar, a space, "100%" or "9.8×" (5), then a gap; at least as wide as its heading
+    cell_w = max(width + 9, *(len(f"{k} {name}") + 2 for k, name, _, _ in LETTERS))
     head = " " * (name_w + 2) + "".join(f"{f'{k} {name}':<{cell_w}}" for k, name, _, _ in LETTERS)
     lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip()]
     for x in result["models"]:
         cells = []
-        for k, _, unit, color in LETTERS:
+        for k, _, shown, color in LETTERS:
             v = x[k]
             full = 0 if v is None else round(width * max(0, min(100, v)) / 100)
+            if v is not None and v < 100:   # full only when perfect (accuracy) or the best (P, T)
+                full = min(full, width - 1)
             bar, rest = "█" * full, "░" * (width - full)
-            num = "-" if v is None else f"{v:.0f}{'%' if unit == '%' else ''}"
+            num = (("-" if v is None else compare.pct(v / 100)) if shown == "A" else times(x.get(shown)))
             bar = f"[{color}]{bar}[/][grey37]{rest}[/]" if markup else bar + rest
-            cells.append(f"{bar} {num:>4}")
+            cells.append(f"{bar} {num:>6}" + " " * (cell_w - width - 7))
         name = f"{x['model']:<{name_w}}"
-        lines.append(f"{rich_escape(name) if markup else name}  " + "  ".join(cells))
+        lines.append((f"{rich_escape(name) if markup else name}  " + "".join(cells)).rstrip())
     return lines
 
 
@@ -116,22 +146,28 @@ def left_out_lines(result):
     return out
 
 
-def _score(v, unit=""):
-    """A is a percentage (%), P and T scores out of 100 (/100: on a log scale, not percentages)."""
-    return "-" if v is None else f"{v:.0f}{unit}"
-
-
 def table(result):
     """(header, rows) for a plain table of the index."""
-    header = ["model", "P parsimony /100", "T speed /100", "A accuracy %", "tokens", "total time",
-              "answers right"]
-    rows = [[x["model"], _score(x["P"]), _score(x["T"]), _score(x["A"], "%"),
-             f"{round(x['tokens']):,}" + (" ★" if x.get("fewest") else ""),
-             fmt_total(x["total_s"]) + (" ★" if x.get("fastest") else ""),
-             f"{x['passed']}/{x['answers']}"] for x in result["models"]]
+    n = result["questions"] or 1
+    header = ["model", "tokens/answer", "time/answer", "answers right", "tok/s", "TTFT s", "memory GB"]
+    rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
+             fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
+             f"{x['passed']}/{x['answers']}", _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
+             _num(x.get("memory_gb"), ".1f")] for x in result["models"]]
     return header, rows
 
 
+def fmt_secs(secs):
+    """Time per answer: "2.5s", "69s", "4m10s"."""
+    if not secs:
+        return "-"
+    return f"{secs:.1f}s" if secs < 10 else f"{secs:.0f}s" if secs < 60 else fmt_total(secs)
+
+
+def _num(v, fmt):
+    return "-" if v is None else format(v, fmt)
+
+
 def fmt_total(secs):
     if not secs:
         return "-"
@@ -147,19 +183,18 @@ def scope_note(result, models):
     if not result["questions"]:
         return "These models have no question in common yet, so there's nothing to compare."
     return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
-            f"A is a percentage. P parsimony and T speed are scores out of 100, not percentages: 100 = the fewest "
-            f"tokens / the fastest total time (★), {POINTS_PER_DOUBLING} points less for each doubling; a better "
-            "model added lowers the others.")
+            "Longer bars are better. P parsimony (tokens) and T time: how many times the leanest / fastest "
+            "model's (1.0×, ★ in the table). A accuracy: answers right.")
 
 
 def for_engine(e, labels=None, packs=None, results_dir=None):
     """(result, models) from a workspace's finished results (results on their way out excluded)."""
     paths = [p for p in compare.default_paths(results_dir or e.results_dir) if not e.superseded(p)]
-    rows, _ = compare.load(paths) if paths else ([], [])
+    rows, infos = compare.load(paths) if paths else ([], [])
     if packs:
         rows = [r for r in rows if r["suite"] in packs]
     models = sorted({r["model"] for r in rows} & set(labels) if labels else {r["model"] for r in rows})
-    return index(rows, models if labels else None), models
+    return index(rows, models if labels else None, memory_gb(infos)), models
 
 
 def markdown(result, models):

tuieval/run_evals.py

@@ -373,11 +373,12 @@ def report_markdown(e, labels=None):
                              + f"; truncated {100 * ev['truncation']:.1f}%"
                              + (f"; consistency {100 * ev['consistency']:.0f}% of {ev['groups']} groups" if ev.get("consistency") is not None else ""))
                 recs = e.records(label, name)
+                from .compare import pct
                 levels = []
                 for lvl in ("easy", "medium", "hard"):
                     x = [r["pass"] for r in recs if r.get("difficulty") == lvl]
                     if x:
-                        levels.append(f"{lvl} {sum(x)}/{len(x)} ({100 * sum(x) / len(x):.0f}%)")
+                        levels.append(f"{lvl} {sum(x)}/{len(x)} ({pct(sum(x) / len(x))})")
                 if levels:
                     lines.append("  - by difficulty: " + ", ".join(levels))
                 for ex in ev.get("critical_examples", []):
@@ -389,7 +390,7 @@ def report_markdown(e, labels=None):
 
 def cmd_pta(argv):
     p = argparse.ArgumentParser(prog="tuieval pta",
-                                description="The PTA index: parsimony (tokens), speed and accuracy per model (0-100, higher is better), as a "
+                                description="The PTA index: parsimony (tokens), time and accuracy per model, as a "
                                             "bar per score for each model and a table, over the questions all the models answered.")
     p.add_argument("--only", help="comma-separated model labels (default: every model with results)")
     p.add_argument("--packs", help="comma-separated packs (default: all)")
@@ -413,7 +414,7 @@ def cmd_pta(argv):
         return
     out.print()
     for line in pta.bars(result, a.width):
-        out.print(line)
+        out.print(line, soft_wrap=True)   # a bar row stays one line (the terminal wraps it if it must)
     header, rows = pta.table(result)
     widths = [max(len(h), *(len(r[i]) for r in rows)) for i, h in enumerate(header)]
     out.print()

tuieval/tui.py

@@ -10,7 +10,7 @@ Run screen:    live reasoning and answer, per-test ✓/✗, TTFT and tokens/s, p
                k skips the current model, c cancels everything, f pauses auto-scroll.
                r shows results at any time (also mid-run); n starts a new run when it ends.
                Enter on a Recent results row shows that answer in full.
-Results:       opens on the PTA index (parsimony, speed and accuracy per model, as bars). Per
+Results:       opens on the PTA index (parsimony, time and accuracy per model, as bars). Per
                question compares each model's tokens and seconds on the same question. ctrl+r
                reloads (it also reloads when a run's pack finishes). Enter on a Failures row or a
                Per question cell shows the answer: grading and checks, reasoning, answer, question.
@@ -19,6 +19,7 @@ Anywhere:      esc/q go back (q quits only on Setup), ctrl+q quits, w lists this
                tunes. Leaving a live run or tune keeps it going; the header shows its progress.
 """
 import argparse
+import collections
 import glob
 import json
 import os
@@ -331,16 +332,15 @@ with the model's reasoning and answer as they stream, then recent results with t
   v  with several models at a time: stream the next one (k skips the one streaming)""",
     "ResultsScreen": """[b]Results[/b]
 
-Opens on the [b]PTA index[/b]: parsimony (tokens), speed and accuracy per model, a bar each (0-100,
-longer = better). [b]Production readiness[/b] gives one verdict per model and use case, and what's
-missing when it's INCONCLUSIVE. The other tabs are the evidence:
-  Scorecard              accuracy, critical failures and truncation per model and pack
-  Speed & tokens         time and tokens per answer (★ = nothing beats it on both accuracy and time)
-  Per question           every question, model by model
-  Is the difference real? whether one model is really better than another, or it's noise
-  Tests that separate    the questions that tell models apart
+  PTA index              parsimony (tokens), time and accuracy per model, a bar each (longer = better;
+                         2.4× = 2.4 times the best), with per-answer numbers, tok/s, TTFT and memory in
+                         the table. Results open here.
+  Production readiness   one verdict per model and use case; the table says why and what to run next
+  Per question           every question, model by model (filters: pack, level, Models disagree)
   Failures               every wrong answer with the grader's reason; enter opens it in full
-  Test quality           tests nobody fails, everybody fails, or that look broken
+
+More in the terminal: tuieval compare (--pairwise: is a difference real), tuieval history,
+tuieval items (test quality).
 
 [b]Keys[/b]  esc back · ctrl+r refresh · w runs""",
     "TuneScreen": """[b]Tuning speed flags[/b]
@@ -2229,7 +2229,7 @@ class RunScreen(Screen):
         if self.running and self.total > done:
             left, rough = self.eta_seconds()
             eta = f" · ETA {'~' if rough else ''}{fmt_secs(left)}"
-        pct = f"{100 * passed / (passed + failed):.0f}%" if passed + failed else "-"
+        pct = compare.pct(passed / (passed + failed)) if passed + failed else "-"
         recent = self.stats[-50:]
         tps = [r["gen_tps"] for r in recent if r.get("gen_tps")]
         ttfts = [r["ttft_s"] for r in recent if r.get("ttft_s") is not None and not r.get("cached_tokens")]
@@ -2347,8 +2347,8 @@ class RunScreen(Screen):
 class ResultsScreen(Screen):
     BINDINGS = [Binding("escape", "app.pop_screen", "Back"), Binding("q", "app.pop_screen", "Back", show=False),
                 Binding("w", "runs", "Runs"), Binding("ctrl+r", "refresh", "Refresh")]
-    SORTS = [("Sort: token gap", "tokens"), ("Sort: time gap", "secs"),
-             ("Sort: most tokens", "most"), ("Sort: pack order", "order")]
+    SORTS = [("By token gap", "tokens"), ("By time gap", "secs"),
+             ("By most tokens", "most"), ("By pack order", "order")]
 
     def check_action(self, action, parameters):
         return bool(self.app.sessions) if action == "runs" else True
@@ -2378,9 +2378,9 @@ class ResultsScreen(Screen):
                 with Horizontal(id="pta-controls"):
                     yield Select([("All packs", "")], id="pta-pack", allow_blank=False, value="")
                     yield Button("Models: all", id="pta-models")
-                yield Static("[dim]P parsimony: tokens used (reasoning and answer). T speed: total time. A accuracy: "
-                             "answers right. Longer bars are better for all three. Verdicts and critical failures "
-                             "are in Production readiness.[/dim]")
+                yield Static("[dim]P parsimony (tokens): tokens used, reasoning and answer. T time: total time. "
+                             "A accuracy: answers right. Verdicts and critical failures are in Production "
+                             "readiness.[/dim]")
                 yield Static(id="pta-note")
                 yield Static(id="pta-bars")
                 yield DataTable(id="pta", zebra_stripes=True, cursor_type="none")
@@ -2393,22 +2393,6 @@ class ResultsScreen(Screen):
                              "measured there, or projected from token counts and that machine's tuned speeds.[/dim]")
                 yield DataTable(id="latency", zebra_stripes=True)
                 yield DataTable(id="readiness-detail", zebra_stripes=True, cursor_type="row")
-            with TabPane("Verdict history"):
-                yield Static("[dim]Every change in a model's verdict, newest first. Kept even after results are rerun "
-                             "or questions change; the evidence behind older verdicts is in results/<model>/history/.[/dim]")
-                yield DataTable(id="history", zebra_stripes=True, cursor_type="row")
-            with TabPane("Scorecard"):
-                yield DataTable(id="scorecard", zebra_stripes=True)
-            with TabPane("Speed & tokens"):
-                yield Select([("As measured (all machines)", "")] +
-                             [(f"{mc.id}{' (this machine)' if i == 0 else ''}", mc.id)
-                              for i, mc in enumerate(self.app.engine.machines())],
-                             id="speed-machine", allow_blank=False, value="")
-                yield Static("[dim]★ = no other model is both more accurate and faster per answer. "
-                             "Memory is the server's resident size (includes GPU-mapped weights on Apple Silicon). "
-                             "On another machine, times are measured there if the model ran there, otherwise "
-                             "projected from each answer's token counts and that machine's tuned speeds.[/dim]")
-                yield DataTable(id="speed", zebra_stripes=True)
             with TabPane("Per question", id="tab-per-question"):
                 with Horizontal(id="pq-controls"):
                     yield Select([("All packs", "")], id="pq-pack", allow_blank=False, value="")
@@ -2416,28 +2400,16 @@ class ResultsScreen(Screen):
                                  id="pq-level", allow_blank=False, value="")
                     yield Select(self.SORTS, id="pq-sort", allow_blank=False, value="tokens")
                     yield Button("Models: all", id="pq-models")
-                    yield Checkbox("Only questions all answered", False, id="pq-shared")
-                    yield Checkbox("Include unfinished", True, id="pq-partial")
+                    yield Checkbox("Answered by all", False, id="pq-shared")
+                    yield Checkbox("Models disagree", False, id="pq-disagree")
+                    yield Checkbox("In progress", True, id="pq-partial")
                 yield Static(id="pq-compare-title")
                 yield DataTable(id="pq-summary", zebra_stripes=True, cursor_type="none")
                 yield Static(id="pq-legend")
                 yield DataTable(id="per-question", zebra_stripes=True, cursor_type="cell")
-            with TabPane("Is the difference real?"):
-                yield DataTable(id="pairwise", zebra_stripes=True)
-            with TabPane("Tests that separate models"):
-                yield DataTable(id="separating", zebra_stripes=True)
-            with TabPane("By difficulty"):
-                yield Static("[dim]Pass rate per difficulty level. The level is your label only; models never see it. "
-                             "A strong model should hold up on HARD, not just EASY.[/dim]")
-                yield DataTable(id="difficulty", zebra_stripes=True)
             with TabPane("Failures"):
                 yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
                 yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
-            with TabPane("Test quality"):
-                yield Static("[dim]no signal: every model passed · suspicious: every model failed (check the test) · "
-                             "inverted: the weakest model beat the strongest · flaky: same model, different results "
-                             "across repeats[/dim]")
-                yield DataTable(id="quality", zebra_stripes=True)
         yield Footer()
 
     def on_mount(self):
@@ -2494,7 +2466,6 @@ class ResultsScreen(Screen):
 
     def fill(self):
         results_dir = self.results_dir or self.app.engine.results_dir
-        self.fill_history()
         paths = compare.default_paths(results_dir)
         self.partial_rows = compare.load_partials(results_dir)
         self.superseded = {p for p in compare.default_paths(results_dir) if self.app.engine.superseded(p)}
@@ -2511,40 +2482,13 @@ class ResultsScreen(Screen):
         smoke = os.path.abspath(results_dir) != os.path.abspath(self.app.engine.results_dir)
         info.update(f"{len(paths)} {'Smoke ' if smoke else ''}result files in {os.path.relpath(results_dir)}/" + banner
                     + "".join(f"\n[yellow]⚠ {n}[/yellow]" for n in notes))
-        header, table = compare.scorecard(rows)
-        t = self.query_one("#scorecard", DataTable)
-        t.add_columns(*header)
-        t.add_rows(table)
         self.rows, self.infos = rows, infos
-        self.fill_speed(self.query_one("#speed-machine", Select).value or "")
-        p = self.query_one("#pairwise", DataTable)
-        p.add_columns("Model A", "Model B", "A − B", "95% CI", "Verdict", "Shared tests")
-        for a, b, diff, lo, hi, call, n in compare.pairwise(rows):
-            color = "green" if call == "clear" else "dim"
-            p.add_row(a, b, f"{100 * diff:+.1f} pts", f"[{100 * lo:+.1f}, {100 * hi:+.1f}]",
-                      f"[{color}]{call}[/{color}]", str(n))
-        split, _ = compare.separating(rows)
-        sep = self.query_one("#separating", DataTable)
-        models = sorted({r["model"] for r in rows})
-        sep.add_columns("Test", *models)
-        for test, d in split:
-            sep.add_row(test, *[f"{sum(d[m])}/{len(d[m])}" if m in d else "-" for m in models])
         f = self.query_one("#failures", DataTable)
         f.add_columns("Model", "Level", "Test", "Reason")
         # compare.failures lists the failed rows in order, so they line up with fail_rows
         self.fail_rows = [r for r in rows if not r["ok"]]
         for model, test, reason, level in compare.failures(rows):
             f.add_row(model, diff_badge(level), test, rich_escape(reason[:130]))
-        dh, dt = compare.by_difficulty(rows)
-        dtab = self.query_one("#difficulty", DataTable)
-        dtab.add_columns("Model", "Pack", *[diff_badge(l) for l in compare.LEVELS])
-        for line in dt:
-            dtab.add_row(*line)
-        qt = self.query_one("#quality", DataTable)
-        qt.add_columns("Flags", "Test", *models)
-        if len(models) >= 2:
-            for test, flags, rates in compare.items(rows):
-                qt.add_row(", ".join(flags), test, *[f"{100 * rates[m]:.0f}%" if m in rates else "-" for m in models])
         self.fill_per_question()
         self.fill_pta()
         self.fill_readiness(results_dir)
@@ -2565,6 +2509,7 @@ class ResultsScreen(Screen):
     @on(Select.Changed, "#pq-sort")
     @on(Checkbox.Changed, "#pq-partial")
     @on(Checkbox.Changed, "#pq-shared")
+    @on(Checkbox.Changed, "#pq-disagree")
     def per_question_changed(self):
         if getattr(self, "rows", None) is not None:
             self.fill_per_question()
@@ -2624,7 +2569,7 @@ class ResultsScreen(Screen):
             self.query_one("#pta-left-out", Static).update("")
             return
         # none picked: every model with enough answers (pta.index leaves out the rest, and says so)
-        result = pta.index(rows, sorted(chosen) or None)
+        result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos))
         self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
         self.query_one("#pta-bars", Static).update("\n".join(pta.bars(result)) if result["questions"] else "")
         self.query_one("#pta-left-out", Static).update(
@@ -2671,6 +2616,8 @@ class ResultsScreen(Screen):
         shown = sorted(chosen) if chosen else sorted({m for t in tests for m in t["cells"]})
         if self.query_one("#pq-shared", Checkbox).value:
             tests = [t for t in tests if all(m in t["cells"] for m in shown)]
+        if self.query_one("#pq-disagree", Checkbox).value:   # some models got it right, others didn't
+            tests = [t for t in tests if len({c["passed"] == c["n"] for c in t["cells"].values()}) > 1]
         self.fill_pq_summary(tests, shown)
         order = {}
         for name, pk in self.app.engine.packs.items():
@@ -2801,38 +2748,6 @@ class ResultsScreen(Screen):
             return
         self.app.push_screen(AnswerScreen(entries, what="repeat"))
 
-    @on(Select.Changed, "#speed-machine")
-    def machine_changed(self, event):
-        if getattr(self, "rows", None) is not None:
-            self.fill_speed(event.value or "")
-
-    def fill_speed(self, machine_id):
-        e = self.app.engine
-        speeds = {}
-        if machine_id:
-            mc = next(x for x in e.machines() if x.id == machine_id)
-            speeds = {m["label"]: (e.serving(m, mc).profile or {}).get("measured") for m in e.cfg["models"]}
-        header, table, _ = compare.speed(self.rows, self.infos, machine_id or None, speeds)
-        sp = self.query_one("#speed", DataTable)
-        sp.clear(columns=True)
-        sp.add_columns(*header)
-        for line in table:
-            sp.add_row(*([f"[yellow]{line[0]}[/yellow]"] + line[1:]))
-
-    def fill_history(self):
-        e = self.app.engine
-        t = self.query_one("#history", DataTable)
-        t.add_columns("When", "Model", "Use case", "Change", "Questions", "Why")
-        rank = {"FAIL": 0, "INCONCLUSIVE": 1, "NO DATA": 1, "PASS": 2}
-        for h in reversed(verdict.load_history(e)):
-            prev, now = h.get("previous"), h["status"]
-            change = f"[{VERDICT_STYLE[now]}]{now}[/]" if not prev else \
-                f"{prev} → [{VERDICT_STYLE[now]}]{now}[/]"
-            if prev and rank.get(now, 1) < rank.get(prev, 1):
-                change += "  [bold red]⚠ worse[/bold red]"
-            qs = "current" if verdict.is_current(h, e) else "[yellow]changed since[/yellow]"
-            t.add_row(h["time"].replace("T", " "), h["model"], h["use_case"], change, qs, "; ".join(h["reasons"])[:120])
-
     def fill_readiness(self, results_dir):
         style = {"PASS": "bold green", "FAIL": "bold red", "INCONCLUSIVE": "yellow", "NO DATA": "dim"}
         matrix = self.query_one("#readiness", DataTable)
@@ -2844,17 +2759,18 @@ class ResultsScreen(Screen):
             return
         help_text.update(self.READINESS_HELP)
         table = verdict.readiness(self.app.engine)
-        short = ["[b]In short[/b]"]
+        short = ["[b]In short[/b]"]   # one line per model; what to run next is in the table below
         for label, sentence, todo in verdict.plain_summary(table):
             short.append(f"  [b]{rich_escape(label)}[/b]: {rich_escape(sentence)}")
-            short += [f"     → {rich_escape(g)}: {rich_escape(what)}" for g, what, _ in todo]
         if any("Certify" in what for _, _, todo in verdict.plain_summary(table) for _, what, _ in todo):
             short.append("  [dim]To run Certify: in Setup tick the model and pack, choose Certify, press s.[/dim]")
         self.query_one("#readiness-short", Static).update("\n".join(short) if table else
                                                           "[dim]No verdicts yet: run something from Setup.[/dim]")
         groups = sorted({g for t in table.values() for g in t})
         matrix.add_columns("Model", *groups)
-        detail.add_columns("Model", "Use case", "Pack", "Verdict", "Why")
+        detail.add_columns("Model", "Use case", "Pack", "Verdict", "Answers right", "Why")
+        right = collections.Counter((r["model"], r["suite"]) for r in self.rows if r["ok"])
+        answered = collections.Counter((r["model"], r["suite"]) for r in self.rows)
         lat = verdict.latency_table(self.app.engine, None, table)
         machine_ids = [mc.id for mc in self.app.engine.machines()]
         lt = self.query_one("#latency", DataTable)
@@ -2871,7 +2787,9 @@ class ResultsScreen(Screen):
             for g, (_, packs) in gs.items():
                 for name, pv in packs.items():
                     if pv.status != "NO DATA":
-                        detail.add_row(label, g, name, f"[{style[pv.status]}]{pv.status}[/]", "; ".join(pv.reasons)[:160])
+                        n = answered[(label, name)]
+                        detail.add_row(label, g, name, f"[{style[pv.status]}]{pv.status}[/]",
+                                       f"{right[(label, name)]}/{n}" if n else "-", "; ".join(pv.reasons)[:160])
 
 
 # ------------------------------------------------------------------ app
@@ -2932,7 +2850,6 @@ class EvalsApp(App):
     #machine { padding: 0 1; height: auto; }
     #tune-status { padding: 0 1; height: auto; min-height: 2; background: $boost; }
     #tune-log { height: 1fr; }
-    #speed-machine { width: 60; margin: 0 1; }
     #results-top { height: auto; }
     #results-info { width: 1fr; height: auto; }
     #toggle-smoke { width: auto; min-width: 32; }
@@ -2943,7 +2860,7 @@ class EvalsApp(App):
     #pta { width: 1fr; height: auto; margin-top: 1; }
     #pta-left-out { height: auto; padding: 1 0 0 0; }
     #pq-controls Checkbox { width: auto; }
-    #pq-controls Select { width: 28; margin-right: 1; }
+    #pq-controls Select { width: 22; margin-right: 1; }
     #pq-models { margin-right: 1; min-width: 20; }
     #pq-compare-title { padding: 0 1; height: auto; }
     #pq-summary { height: auto; max-height: 10; }

tuieval/tune.py

@@ -60,6 +60,7 @@ SWAP_NOTABLE = 256 * 2**20  # swap worth a warning (bytes); rejecting for swap n
 ANSWER_TOKENS = 1024       # answer length the projected objective scores ([tune] answer_tokens)
 PRESSURE_CRITICAL = 4      # macOS memory pressure level that rejects a candidate
 PRESSURE_EVERY_S = 2
+VRAM_MARGIN_GB = 0.25     # a discrete GPU this close to full could run out on a longer prompt
 GPU_MARGIN_GB = 0.75      # a candidate whose GPU allocation comes this close to the residency limit
                           # is rejected even before it stalls: servers grow as requests arrive
 REQUEST_LIMIT_S = 600     # a tuning request taking longer fails the candidate ([tune] request_timeout_s)
@@ -556,6 +557,14 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                     r.facts["gpu_settled_gb"] = round(settled, 1)
                 if info.get("kernel_share_max") is not None:
                     r.facts["kernel_share"] = info["kernel_share_max"]
+                mc = eng.machine()
+                if mc.discrete:   # VRAM is the limit there: its own memory, nothing else can use it
+                    vram = machines.vram_used_gb()
+                    if vram is not None:
+                        r.facts["vram_used_gb"] = round(vram, 1)
+                        if vram > mc.gpu_limit_gb - VRAM_MARGIN_GB and not r.error:
+                            r.error = (f"VRAM nearly full: {vram:.1f} of {mc.gpu_limit_gb:.0f} GB in use "
+                                       "(a longer prompt could run out)")
                 limit = eng.gpu_residency_gb()
                 if settled and settled > limit - GPU_MARGIN_GB and not r.error:
                     r.error = (f"GPU allocation settled at {settled:.1f} GB, within {GPU_MARGIN_GB} GB of the "

tuieval/verdict.py

@@ -259,7 +259,8 @@ def _plain_fail(v):
     if "cut off" in text:
         return "too many answers cut off by max_tokens"
     if "accuracy" in text and ev.get("trials"):
-        return f"{100 * ev['accuracy']:.0f}% right, under the bar"
+        from .compare import pct   # local: compare imports nothing from here, but keep verdict light
+        return f"{pct(ev['accuracy'])} right, under the bar"
     return text.split(" (")[0]
 
 

{tuieval-0.2.0.dev56.dist-info → tuieval-0.2.0.dev65.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev56
+Version: 0.2.0.dev65
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -51,7 +51,7 @@ Whatever runs your models, the first steps are short:
 | You have | Do this |
 |---|---|
 | **LM Studio, Ollama or vLLM** | Start its server and load a model, then `tuieval init my-evals`: it finds the model. Later: `tuieval add` lists new ones. |
-| **GGUF files** | Install llama.cpp (`brew install llama.cpp`), then `tuieval add ~/models/Some-Model-Q4_K_M.gguf`. tuieval starts `llama-server` for it, sized to your machine. |
+| **GGUF files** | Install llama.cpp (`brew install llama.cpp`, or a CUDA/ROCm build on Linux), then `tuieval add ~/models/Some-Model-Q4_K_M.gguf`. tuieval starts `llama-server` for it, sized to your Mac's memory or your Nvidia/AMD card's VRAM. |
 | **An OpenRouter key** | `export OPENROUTER_API_KEY=…`, then `tuieval run --tier smoke --only openrouter:<model id>`. |
 
 Stuck? `tuieval doctor` checks servers, models, packs and keys, and says what to fix.
@@ -76,7 +76,7 @@ Afterwards, every answer keeps its reasoning. Enter on a result opens it with th
 - **Verdicts you can act on.** A pack passes only with zero critical failures over enough trials, and an accuracy whose 95% lower bound clears your bar. *INCONCLUSIVE* says what evidence is missing.
 - **Two tiers:** *Screen* a sample of every pack to drop weak models fast, then *Certify* finalists on every test with repeats. Certification reuses the screening answers.
 - **Honest numbers:** every request is a fresh single-turn conversation, with prompt caching off and repeat rounds in different orders. Each answer records what was sent, and the run warns about reused prompts or identical repeats.
-- **Per-machine speed.** A fit check picks the largest context that fits your Mac's GPU memory, `tuieval tune` finds the fastest speed-only server flags (with a guard that rejects flags that change answers), and readiness includes a *Fast enough?* table per machine, measured or projected.
+- **Per-machine speed.** A fit check picks the largest context that fits your Mac's GPU memory or your Nvidia/AMD card's VRAM, `tuieval tune` finds the fastest speed-only server flags (with a guard that rejects flags that change answers), and readiness includes a *Fast enough?* table per machine, measured or projected.
 - **Tests that check themselves.** Every test carries a reference answer and known-wrong answers, and `tuieval selftest` checks the grader accepts the first and rejects the second, and that each gate is reachable at all.
 - **History.** Verdict changes are appended to `results/verdicts.jsonl`, and replaced results are moved to `history/`, never overwritten.
 
@@ -144,7 +144,7 @@ See **[docs/models.md](docs/models.md)** for servers, OpenRouter endpoint pinnin
 1. **Pick packs and models** (space ticks; type to filter models by label or tag). Each model shows a short code per use case, e.g. `C✓ S?` (✓ pass, ✗ fail, ? inconclusive; grey = from earlier results). To run only some tests of a pack, highlight it and press `e`. Runs of different tests add up, and the pack can PASS once all its tests have run.
 2. **Pick a tier** (Smoke to check setup, Screen, Certify) and **press `s`**. The line above the buttons shows how many answers that is and roughly how long it will take. For each model, tuieval starts its server (or uses a running one), checks the right model is loaded, runs every selected pack, then stops it.
 3. **Watch the run:** progress with ETA, live score, tok/s, TTFT and memory, the current test with reasoning and answer side by side, recent results with the grader's reason. `k` skips a model, `c` cancels (finished work is kept and resumes next time). `n` sets up more evals while it runs: add them to this run (same tier) or queue them as the next run.
-4. **Press `r` for results:** production readiness, verdict history, scorecard, speed & tokens (★ marks models nothing beats on both accuracy and time), per question, is the difference real?, tests that separate models, by difficulty, failures, and test quality.
+4. **Press `r` for results:** the PTA index (parsimony (tokens), time and accuracy as bars), production readiness, per question, and failures. `tuieval compare`, `tuieval history` and `tuieval items` have the rest: whether a difference is real, verdict history and test quality.
 
 Other keys: `t` tunes the ticked models' speed flags, `a` adds a model, `m` scans for GGUFs, `p` saves or loads a preset, `x` hides a model. `?` explains any screen.
 
@@ -176,7 +176,7 @@ tuieval help
 
 - **Start with Production readiness.** A single critical failure already means FAIL, whatever the accuracy: a model that breaks a hard rule once in 300 answers will do it in production. The Failures tab and `tuieval report` list exactly which answers failed.
 - **INCONCLUSIVE is not "nearly passed".** It says what's missing (usually a Certify run, or more trials).
-- **Trust "Is the difference real?" over raw percentages.** Differences of one or two tests are usually noise.
+- **Trust `tuieval compare --pairwise` over raw percentages.** Differences of one or two tests are usually noise.
 - **Read `trunc` before accuracy.** A model that runs out of tokens isn't wrong; it's thinking too long for the budget.
 - **Watch TTFT for interactive use.** A model that is 5% more accurate but takes 3 s longer to start answering may be the worse choice.
 

{tuieval-0.2.0.dev56.dist-info → tuieval-0.2.0.dev65.dist-info}/RECORD

@@ -1,25 +1,25 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=w3V6noQtCnYcwkOc0Hi3EmCrxSS1deuXpcG9aisclJI,535
+tuieval/_version.py,sha256=QkealilRB02_mYePp9PrQfXW0yGA1qHLQzzPeETGVN4,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
-tuieval/cli.py,sha256=gzGBzhqvR0WKJQQNHB8CudpgViIeIyawrNWa54whmoI,4502
+tuieval/cli.py,sha256=fn6z0cupM-aW3fY5fFgnS-HmOQmMzlDg-U76Y6LgL3o,4510
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
-tuieval/compare.py,sha256=SHN_o_nlppFcHMBqBZRITIkeQDPaR_JxhUZItQLMBwo,26270
-tuieval/doctor.py,sha256=y1C_ggjd8oikXTrovgTBf26wjThP8EoVkfTVeO_n5dQ,6070
-tuieval/engine.py,sha256=0PuHFus1k02qAY8nlsI6z5iQo7IT82bMwmHDZT6fGtU,114784
+tuieval/compare.py,sha256=T6a_PC7z-jHEvw09KEceo2dxoLMi-9cHYPQs4L6Ctig,26997
+tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
+tuieval/engine.py,sha256=SsXqR1ydn827qBX8NoawyT2roNuelmrZB-hOuyGSesY,116514
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
-tuieval/machines.py,sha256=tdRrn3WNOmCHwfcxcWz1NuAuOJdI7WnJFR1y6xgF89c,13512
+tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
-tuieval/pta.py,sha256=KkrOxE1hTDwu1435IrzMzIysrgVUvFuKVo786VlRyK4,9056
+tuieval/pta.py,sha256=6HwPMvTTJFoZjO-2ZurucJrp1itIzpDTKL1xvYP9W0w,10888
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=5bQEqU533XYiLze9dYcl72c5GSIpohMj1eqJrePRwnc,46450
+tuieval/run_evals.py,sha256=1zNkCdFt744cqpebqiyj-sZrbOI22c6IYBcXk9koz20,46538
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=xAQnR270m7x-rfTdK7LBPMM9MEQC37zndD1TOTq8EUU,166103
-tuieval/tune.py,sha256=jXwYuSjfdqbLCE3WonP2xaUHwk_FnhI67twB1BaHggI,38992
-tuieval/verdict.py,sha256=Muo1a05mCv781MlEJsXwDsGaKbRi0JS4Lv8HdvK_Aco,19054
+tuieval/tui.py,sha256=lrIv4z21s9OG809706Z-lE2-TZ6gg8z0AjaJ_zW7zkw,161021
+tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
+tuieval/verdict.py,sha256=VB-0LTxR9y8bJHT9RicxtE9K7cUHBdCu40hOCrsbmA4,19150
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
 tuieval/yamlout.py,sha256=r1IokjLqW-Ztpg1fSn33ZW3VK00v1VvxymmU5Qy7GEw,606
@@ -46,8 +46,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev56.dist-info/METADATA,sha256=nUWInJiqgtyGg3hiyaYIUuGD-BYOzoyqi3kRoSloRPk,13881
-tuieval-0.2.0.dev56.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev56.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev56.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev56.dist-info/RECORD,,
+tuieval-0.2.0.dev65.dist-info/METADATA,sha256=Wd-olyVQVqZyQTb817isEBh7GUdLGbBQDxy2Ioc7iBw,13996
+tuieval-0.2.0.dev65.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev65.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev65.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev65.dist-info/RECORD,,