tuieval 0.2.0.dev25__py3-none-any.whl → 0.2.0.dev32__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev25'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev25')
+__version__ = version = '0.2.0.dev32'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev32')
 
 __commit_id__ = commit_id = None

tuieval/compare.py

@@ -37,7 +37,8 @@ def _load_native(data):
             "settings": data.get("settings", {}), "load_s": run.get("load_s"),
             "peak_rss_mb": run.get("peak_rss_mb"), "pack": pack["name"], "pack_fp": pack["fingerprint"],
             # hosted (OpenRouter) models: the provider endpoint and its declared quantization
-            "endpoint": run.get("endpoint"), "quantization": run.get("quantization")}
+            "endpoint": run.get("endpoint"), "quantization": run.get("quantization"),
+            "loaded_alongside": run.get("loaded_alongside") or []}
     rows = []
     for r in data["results"]:
         rows.append({
@@ -54,6 +55,8 @@ def _load_native(data):
             "severity": r.get("severity"), "group": r.get("group"), "repeat": r.get("repeat", 0),
             "difficulty": r.get("difficulty", "unrated"),
             "machine": r.get("machine") or run.get("machine"), "prompt_tokens": r.get("prompt_tokens"),
+            # other models served at the same time (parallel_models): its speed was shared with them
+            "alongside": r.get("ran_alongside") or [],
         })
     return rows, info
 
@@ -210,6 +213,13 @@ def timing(r, machine=None, speeds=None):
     return None, None, None, None, None
 
 
+def alongside_note(rows):
+    """' · n/N ran alongside others' when some answers ran while other models were served (their
+    speed was shared with them), else ''."""
+    shared = sum(bool(r.get("alongside")) for r in rows)
+    return f" · {shared}/{len(rows)} ran alongside others" if shared else ""
+
+
 def speed(rows, infos, machine=None, speeds=None):
     """(header, table, frontier): speed and token use per model.
 
@@ -227,7 +237,9 @@ def speed(rows, infos, machine=None, speeds=None):
     load = {}
     for i in infos:
         cur = load.setdefault(i["label"], {"load_s": None, "peak_rss_mb": None, "tags": i["tags"],
-                                           "think": i["settings"].get("enable_thinking"), "served": set()})
+                                           "think": i["settings"].get("enable_thinking"), "served": set(),
+                                           "loaded_alongside": False})
+        cur["loaded_alongside"] |= bool(i.get("loaded_alongside"))
         if i.get("endpoint"):
             q = i.get("quantization")
             cur["served"].add(f"{'undeclared' if q in (None, 'unknown') else q} via {i['endpoint']}")
@@ -247,6 +259,7 @@ def speed(rows, infos, machine=None, speeds=None):
         here = machine is None or machine in where   # load time and memory only where it ran
         if machine is None:
             kind = "measured on " + (", ".join(where) or "?")
+        kind += alongside_note(v)
         table.append([
             "★" if m in frontier else "", m, "; ".join(sorted(info.get("served") or [])) or "-",
             ",".join(info.get("tags") or []),
@@ -256,7 +269,8 @@ def speed(rows, infos, machine=None, speeds=None):
             _fmt(median(x[3] for x in t), ".0f"), _fmt(median(r["tokens"] for r in v), ".0f"),
             _fmt(median(r["reasoning_tokens"] for r in v), ".0f"), _fmt(median(r["answer_tokens"] for r in v), ".0f"),
             f"{sum(r['tokens'] for r in v) / passes:,.0f}" if passes else "-",
-            _fmt(info.get("load_s") if here else None, ".0f"),
+            _fmt(info.get("load_s") if here else None, ".0f")
+            + (" (alongside others)" if here and info.get("load_s") and info.get("loaded_alongside") else ""),
             _fmt(info["peak_rss_mb"] / 1024 if info.get("peak_rss_mb") and here else None, ".1f"),
         ])
     return header, table, frontier

tuieval/engine.py

@@ -229,6 +229,7 @@ class Serving:
     identity: dict             # output-affecting settings, part of the results fingerprint
     profile: dict | None = None
     mtp_layers: int = 0
+    need_gb: float | None = None   # estimated GPU memory at ctx (from the fit check); None when unknown
 
 
 def infer_server(cfg, model):
@@ -664,6 +665,20 @@ def holder_text(holder):
             "One eval or tune per machine: this starts when it ends")
 
 
+class Lane:
+    """One model being run: its own request stream, stall reason and skip switch, so models run
+    side by side (parallel_models) stop independently. Runs and tunes in one lane use the engine's
+    main lane."""
+    def __init__(self, label=None):
+        self.label = label
+        self.stream = None
+        self.stall = None         # set by the stall watcher: why this model's server stalled
+        self.skip = threading.Event()
+        self.procs = []
+        self.port = None
+        self.seen = set()         # other models served during this model's current load or request
+
+
 class Engine:
     def __init__(self, on_event=None, root=None, models_path=None, packs_dir=None,
                  results_dir=None, log_dir=None):
@@ -677,11 +692,14 @@ class Engine:
         self.pack_errors = []
         self.packs = packs_mod.load_packs(self.packs_dir, self.pack_errors)
         self._cancel = threading.Event()
-        self._skip = threading.Event()
-        self._stall = None        # set by the stall watcher: why the current server stalled
+        self._main_lane = Lane()
+        self._here = threading.local()   # .lane: the lane this thread runs
+        self._lanes = {}          # label -> Lane of each model being served now
+        self._lanes_lock = threading.Lock()
         self._procs = []
         self._stop_lock = threading.Lock()
-        self._stream = None
+        self._run_lock = threading.Lock()
+        self._run = None          # the run going: {"jobs", "pending", "open", "machine", "cond"} (add_jobs)
         self._header_cache = {}
         self._serving = {}
         self.tuning_dir = os.path.join(root, "tuning")
@@ -690,6 +708,34 @@ class Engine:
         self._catalogues = {}
         self._add_remote_models()
 
+    # ---- the lane this thread runs (see Lane)
+    def _lane(self):
+        return getattr(self._here, "lane", None) or self._main_lane
+
+    def _all_lanes(self):
+        with self._lanes_lock:
+            return [self._main_lane, *self._lanes.values()]
+
+    @property
+    def _stream(self):
+        return self._lane().stream
+
+    @_stream.setter
+    def _stream(self, value):
+        self._lane().stream = value
+
+    @property
+    def _stall(self):
+        return self._lane().stall
+
+    @_stall.setter
+    def _stall(self, value):
+        self._lane().stall = value
+
+    @property
+    def _skip(self):
+        return self._lane().skip
+
     def reload(self):
         self._serving = {}
         self.cfg = load_config(self.models_path)
@@ -1150,6 +1196,30 @@ class Engine:
         """models.toml [machines.<id>] (optional): memory_headroom_gb, name."""
         return self.cfg.get("machines", {}).get(machine_id or self.machine().id, {})
 
+    def parallel_models(self):
+        """How many models a run serves at a time on this machine: models.toml [machines.<id>]
+        parallel_models (default 1)."""
+        try:
+            return max(1, int(self.machine_settings().get("parallel_models", 1)))
+        except (TypeError, ValueError):
+            return 1
+
+    def memory_need_gb(self, m):
+        """GPU memory a model takes while served here: 0 for one we don't start (a hosted API or a
+        server already running), the fit check's estimate for a local GGUF, else None (unknown)."""
+        if not self.cfg["servers"][m["server"]].get("cmd"):
+            return 0.0
+        return self.serving(m).need_gb
+
+    def memory_available_gb(self):
+        """GPU memory models may take here (what the fit check sizes contexts against)."""
+        mc = self.machine()
+        return mc.gpu_limit_gb - self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
+
+    def runs_alone(self, m):
+        """A server whose before_start step may stop other servers never runs next to another model."""
+        return bool(self.cfg["servers"][m["server"]].get("before_start"))
+
     def gpu_residency_gb(self, machine=None):
         """GPU memory this machine's driver keeps resident without churning (see machines.py)."""
         mc = machine or self.machine()
@@ -1180,7 +1250,7 @@ class Engine:
         kv = m.get("kv_type") or server.get("kv_type")
         want = min(x for x in (server.get("max_ctx"), m.get("ctx"), m.get("max_context"), 10**9) if x)
         want = None if want == 10**9 else want
-        ctx, fits, note, mtp, size = want, True, "", 0, None
+        ctx, fits, note, mtp, size, need = want, True, "", 0, None, None
         path = expand(m["model"])
         if server.get("model_is_path") and os.path.isfile(path):
             try:
@@ -1192,7 +1262,7 @@ class Engine:
                     headroom = self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
                     f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=headroom, extra_bytes=extra,
                                      want_ctx=want)
-                    ctx, fits, note = f.max_ctx, f.fits, f.note
+                    ctx, fits, note, need = f.max_ctx, f.fits, f.note, f.need_gb
             except (OSError, ValueError) as e:
                 note = f"couldn't read the GGUF header ({e}); context not checked"
         profile, perf, source = None, [], "n/a"
@@ -1221,7 +1291,7 @@ class Engine:
             # resident), so both are part of what a result is valid for.
             identity["machine"] = mc.id
             identity["perf"] = perf
-        sv = Serving(mc, ctx, kv, fits, note, perf, source, identity, profile, mtp)
+        sv = Serving(mc, ctx, kv, fits, note, perf, source, identity, profile, mtp, need)
         self._serving[key] = sv
         return sv
 
@@ -1289,16 +1359,21 @@ class Engine:
     # ---- control
     def cancel(self):
         self._cancel.set()
-        self._abort()
-
-    def skip_model(self):
-        self._skip.set()
-        self._abort()
-
-    def _abort(self):
-        if self._stream:
-            self._stream.close()
-        for p in list(self._procs):
+        self._abort(self._all_lanes(), list(self._procs))
+
+    def skip_model(self, label=None):
+        """Skip a model being run (label), or every one being run now."""
+        with self._lanes_lock:
+            lanes = [self._lanes[label]] if label in self._lanes else list(self._lanes.values()) or [self._main_lane]
+        for lane in lanes:
+            lane.skip.set()
+        self._abort(lanes, [p for lane in lanes for p in lane.procs] if self._lanes else list(self._procs))
+
+    def _abort(self, lanes, procs):
+        for lane in lanes:
+            if lane.stream:
+                lane.stream.close()
+        for p in procs:
             self._stop(p, grace=10)
 
     def _check(self):
@@ -1322,6 +1397,7 @@ class Engine:
         STALL_WINDOW_S, the GPU driver is churning memory: record why and abort the request."""
         mc = self.machine()
         limit = self.gpu_residency_gb(mc)
+        lane = self._lane()
         samples = []   # (time, user, kernel)
 
         def run():
@@ -1343,15 +1419,15 @@ class Engine:
                     continue
                 share = dk / (du + dk)
                 info["kernel_share_max"] = max(info.get("kernel_share_max") or 0, round(share, 2))
-                if share > self.STALL_KERNEL_SHARE and (du + dk) / dt > self.STALL_MIN_BUSY and not self._stall:
+                if share > self.STALL_KERNEL_SHARE and (du + dk) / dt > self.STALL_MIN_BUSY and not lane.stall:
                     over = f"{alloc:.1f} GB of GPU memory allocated; " if alloc else ""
-                    self._stall = (f"{m['label']} stalled in the GPU driver on {mc.id}: {100 * share:.0f}% of its "
+                    lane.stall = (f"{m['label']} stalled in the GPU driver on {mc.id}: {100 * share:.0f}% of its "
                                    f"CPU time went to the kernel for {self.STALL_WINDOW_S}s while the GPU waited "
                                    f"({over}this machine keeps ~{limit:.0f} GB resident). "
                                    "Lower its GPU memory or retune it.")
-                    self.emit("server_stall", label=m["label"], message=self._stall)
-                    if self._stream:
-                        self._stream.close()
+                    self.emit("server_stall", label=m["label"], message=lane.stall)
+                    if lane.stream:
+                        lane.stream.close()
         threading.Thread(target=run, daemon=True).start()
 
     # ---- processes
@@ -1360,6 +1436,7 @@ class Engine:
         p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.DEVNULL, stdout=out,
                              stderr=subprocess.STDOUT, start_new_session=True)
         self._procs.append(p)
+        self._lane().procs.append(p)
         return p
 
     def _stop(self, p, grace=30):
@@ -1381,6 +1458,9 @@ class Engine:
                     p.wait()
             if p in self._procs:
                 self._procs.remove(p)
+            for lane in self._all_lanes():
+                if p in lane.procs:
+                    lane.procs.remove(p)
 
     def _tail(self, path, label, stop):
         def run():
@@ -1472,45 +1552,40 @@ class Engine:
         return any(self.cfg["servers"][j.model["server"]].get("cmd") for j in jobs if j.status == "waiting")
 
     # ---- running
-    def run(self, jobs):
-        """Run the jobs (blocking). Jobs of the same model share one server start."""
+    def run(self, jobs, parallel=None):
+        """Run the jobs (blocking). Jobs of the same model share one server start. parallel: how many
+        models are served at a time (default: parallel_models for this machine). add_jobs() can add
+        more while it runs; they run after the ones before them."""
         self._cancel.clear()
         self.emit("queue_started", jobs=jobs)
         models = sorted({j.label for j in jobs if j.status == "waiting"})
+        machine = bool(jobs) and self.needs_machine(jobs)
         lock = (self.machine_lock(f"{jobs[0].tier} run: {', '.join(models[:3])}"
                                   + (f" +{len(models) - 3}" if len(models) > 3 else ""))
-                if jobs and self.needs_machine(jobs) else contextlib.nullcontext())
+                if machine else contextlib.nullcontext())
+        order = []
+        for j in jobs:
+            if j.label not in order:
+                order.append(j.label)
+        groups = [(label, [j for j in jobs if j.label == label and j.status == "waiting"]) for label in order]
+        with self._run_lock:
+            self._run = {"jobs": jobs, "pending": [g for g in groups if g[1]], "open": True, "machine": machine,
+                         "cond": threading.Condition(self._run_lock)}
         try:
             with lock:
-                order = []
-                for j in jobs:
-                    if j.label not in order:
-                        order.append(j.label)
-                for label in order:
-                    mine = [j for j in jobs if j.label == label and j.status == "waiting"]
-                    if not mine:
-                        continue
-                    self._skip.clear()
-                    try:
-                        try:
-                            self._run_model(mine)
-                        except (Cancelled, ModelFailed):
-                            raise
-                        except Exception as e:  # never let one model's surprise kill the whole queue
-                            raise ModelFailed(f"unexpected error: {e!r}") from e
-                    except ModelFailed as e:
-                        for j in mine:
-                            if j.status in ("waiting", "loading", "running"):
-                                j.status = "skipped" if "by user" in str(e) else "failed"
-                                j.note, j.finished = str(e).splitlines()[0], time.time()
-                                self.emit("job_done", job=j)
-                        self.emit("model_failed", label=label, message=str(e))
+                self._run_lanes(max(1, int(parallel or self.parallel_models())))
+            if self._cancel.is_set():
+                raise Cancelled()
         except Cancelled:
+            with self._run_lock:
+                self._run["open"] = False
             for j in jobs:
                 if j.status in ("waiting", "loading", "running"):
                     j.status, j.note = "skipped", "cancelled (progress kept)" if j.done else "cancelled"
                     self.emit("job_done", job=j)
         finally:
+            with self._run_lock:
+                self._run["open"] = False
             changes = []
             if any(j.status == "done" and j.started and j.tier != "smoke" for j in jobs):
                 try:
@@ -1522,6 +1597,145 @@ class Engine:
                 self.emit("verdicts", changes=changes)
             self.emit("queue_done", jobs=jobs, cancelled=self._cancel.is_set())
 
+    def add_jobs(self, new):
+        """Add planned jobs to the run going on, after its other jobs. Returns (added, why not): a
+        pack already waiting or running in it isn't added twice, and nothing is added once the run is
+        finishing or cancelled, or when it holds no machine lock (hosted only) and these need one.
+        Runs on the caller's thread, so the caller shows what was added (no event)."""
+        with self._run_lock:
+            r = self._run
+            if not r or not r["open"] or self._cancel.is_set():
+                return [], "the run is finishing"
+            if not r["machine"] and self.needs_machine(new):
+                return [], "the run going uses only hosted models, and these need this machine"
+            busy = {j.key for j in r["jobs"] if j.status in ("waiting", "loading", "running")}
+            added = [j for j in new if j.status == "waiting" and j.key not in busy]
+            r["jobs"].extend(added)
+            for j in added:   # a model still waiting takes them on; otherwise it comes again at the end
+                group = next((g for g in r["pending"] if g[0] == j.label), None)
+                if group:
+                    group[1].append(j)
+                else:
+                    r["pending"].append((j.label, [j]))
+            r["cond"].notify_all()
+        return added, None if added else "everything picked is already in the run"
+
+    def _run_lanes(self, parallel):
+        """Run each model's jobs in its own lane, up to `parallel` at a time, in order: the next
+        model starts once a lane is free, nothing running stops it (runs_alone), and its memory fits
+        next to the ones running (memory_need_gb; an unknown need is left to the setting). Lanes
+        wait for jobs added meanwhile (add_jobs) until every model is done."""
+        pending, running, told = self._run["pending"], {}, set()
+        cond = self._run["cond"]
+
+        def blocked(label):
+            """Why the next model can't start now (None: it can)."""
+            if not running:
+                return None
+            if len(running) >= parallel:
+                return ""
+            m = self.model(label)
+            if self.runs_alone(m) or any(self.runs_alone(self.model(l)) for l in running):
+                alone = label if self.runs_alone(m) else next(l for l in running if self.runs_alone(self.model(l)))
+                return (f"{alone}'s server runs a before_start step that may stop other servers, so it runs "
+                        "on its own")
+            need = self.memory_need_gb(m)
+            used = sum(n for n in running.values() if n)
+            room = self.memory_available_gb()
+            if need and used + need > room:
+                return (f"needs ~{need:.0f} GB; {', '.join(running)} take ~{used:.0f} of {room:.0f} GB, so it "
+                        "starts when there's room")
+            return None
+
+        def lane():
+            while True:
+                mine, note = None, None
+                with cond:   # events go out after the lock is released: the UI may be waiting for it (add_jobs)
+                    if self._cancel.is_set() or not pending and not running:
+                        self._run["open"] = False   # nothing more can be added
+                        cond.notify_all()
+                        return
+                    # in order, except that a model added again while it runs waits for itself to end
+                    # without holding up the ones behind it
+                    nxt = next((i for i, g in enumerate(pending) if g[0] not in running), None)
+                    label = pending[nxt][0] if nxt is not None else None
+                    why = blocked(label) if label else ""   # none can start: a model still runs, more may come
+                    if why is None:
+                        label, mine = pending.pop(nxt)
+                        running[label] = self.memory_need_gb(self.model(label))
+                    elif why and label not in told:
+                        told.add(label)
+                        note = f"{label} waits: {why}"
+                    else:
+                        cond.wait(1.0)
+                if note:
+                    self.emit("model_waiting", label=label, message=note)
+                if mine is None:
+                    continue
+                try:
+                    self._run_lane(label, mine)
+                finally:
+                    with cond:
+                        running.pop(label, None)
+                        cond.notify_all()
+
+        if parallel == 1:
+            lane()
+            return
+        threads = [threading.Thread(target=lane, name=f"lane-{i + 1}") for i in range(parallel)]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+    def _run_lane(self, label, mine):
+        """One model's jobs, in this thread's own lane. Other models served meanwhile are noted on
+        each answer (ran_alongside), since they share the machine's GPU and memory bandwidth."""
+        lane = Lane(label)
+        m = mine[0].model
+        with self._lanes_lock:
+            for other in self._lanes.values():
+                other.seen.add(label)
+            lane.seen = set(self._lanes)
+            if self.cfg["servers"][m["server"]].get("cmd"):
+                lane.port = self._lane_port(m)
+            self._lanes[label] = lane
+        self._here.lane = lane
+        try:
+            try:
+                self._run_model(mine)
+            except (Cancelled, ModelFailed):
+                raise
+            except Exception as e:  # never let one model's surprise kill the whole queue
+                raise ModelFailed(f"unexpected error: {e!r}") from e
+        except ModelFailed as e:
+            for j in mine:
+                if j.status in ("waiting", "loading", "running"):
+                    j.status = "skipped" if "by user" in str(e) else "failed"
+                    j.note, j.finished = str(e).splitlines()[0], time.time()
+                    self.emit("job_done", job=j)
+            self.emit("model_failed", label=label, message=str(e))
+        except Cancelled:
+            pass   # run() marks what's left
+        finally:
+            self._here.lane = None
+            with self._lanes_lock:
+                self._lanes.pop(label, None)
+
+    def _lane_port(self, m):
+        """The server's port, or a free one when another model being served has it (call with
+        _lanes_lock held)."""
+        port = self.cfg["servers"][m["server"]].get("port")
+        if port is None or not any(l.port == port for l in self._lanes.values()):
+            return port
+        taken = {l.port for l in self._lanes.values()}
+        while True:
+            with socket.socket() as s:
+                s.bind(("127.0.0.1", 0))
+                free = s.getsockname()[1]
+            if free not in taken:
+                return free
+
     @contextlib.contextmanager
     def serve(self, m, perf_args=None, port=None, log_name=None):
         """Start a model's server (or use the running one), wait until it serves the expected name,
@@ -1649,11 +1863,16 @@ class Engine:
         for j in jobs:
             j.status = "loading"
             self.emit("job_update", job=j)
-        with self.serve(m) as (base_url, info):
+        lane = self._lane()
+        with self.serve(m, port=lane.port) as (base_url, info):
+            with self._lanes_lock:
+                loaded_alongside = sorted(lane.seen)   # served while this one loaded: its load time is shared
+
             def run_info():
                 return {"load_s": info["load_s"], "peak_rss_mb": info["peak_mb"](),
                         "server_facts": info["facts"] or None, "gpu_peak_gb": info.get("gpu_peak_gb"),
-                        "kernel_share_max": info.get("kernel_share_max"), **serving_info}
+                        "kernel_share_max": info.get("kernel_share_max"),
+                        **({"loaded_alongside": loaded_alongside} if loaded_alongside else {}), **serving_info}
             failed_in_row = 0
             for j in jobs:
                 self._check()
@@ -1736,7 +1955,7 @@ class Engine:
         return tests
 
     # ---- sittings: a pack run over several sessions (stopped and resumed) keeps its total time
-    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "kernel_share_max", "machine")
+    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "kernel_share_max", "machine", "loaded_alongside")
 
     def _sitting(self, job, info):
         now = time.time()
@@ -1811,6 +2030,9 @@ class Engine:
         for k in ("peak_rss_mb", "gpu_peak_gb", "kernel_share_max"):
             if peak(k) is not None:
                 out[k] = peak(k)
+        along = sorted({l for s in sittings for l in s.get("loaded_alongside") or []})
+        if along:
+            out["loaded_alongside"] = along
         return out
 
     def _run_job(self, job, base_url, run_info, alive=lambda: True):
@@ -1862,6 +2084,9 @@ class Engine:
                 if (test["id"], rep) in done_keys:
                     continue
                 self._check()
+                lane = self._lane()
+                with self._lanes_lock:
+                    lane.seen = set(self._lanes) - {job.label}
                 self.emit("request_started", job=job, test=test["description"], test_id=test["id"],
                           has_image=bool(test.get("image")), repeat=rep, difficulty=test.get("difficulty", "unrated"))
                 waits = list(self.SERVER_RETRY_WAITS_S)
@@ -1870,7 +2095,7 @@ class Engine:
                     try:
                         body = self._body(job, test, rep)
                         res = client.stream_chat(base_url, body,
-                                                 lambda kind, text: self.emit("delta", stream=kind, text=text),
+                                                 lambda kind, text: self.emit("delta", stream=kind, text=text, label=job.label),
                                                  timeout=timeout, stream=self._stream, headers=headers)
                     except client.Cancelled:
                         self._check()
@@ -1917,6 +2142,8 @@ class Engine:
                        **({"checks": g["checks"]} if "checks" in g else {}),
                        "machine": self.machine().id,
                        **({"quantization": ep.get("quantization")} if ep else {}),
+                       # other models served while this answer ran: they shared the machine, so its speed did too
+                       **({"ran_alongside": sorted(lane.seen)} if lane.seen else {}),
                        "sent": "+".join(msg["role"] for msg in body["messages"]),  # proof: single turn
                        **({"seed": body["seed"]} if "seed" in body else {}), **meta}
                 # The same long answer twice for one question, with sampling on, means a cached response

tuieval/run_evals.py

@@ -15,6 +15,7 @@
     tuieval run --tier smoke --only my-model      # 3 tests per pack, 1 repeat: check paths and flags
     tuieval run --tier smoke --only openrouter:qwen/qwen3-32b   # any OpenRouter model, no models.toml edit
     tuieval run --dry-run                         # print the plan and server commands only
+    tuieval run --parallel 2                      # serve two models at a time (parallel_models in models.toml)
     tuieval add ~/models/New-Model-Q4_K_M.gguf    # register a model (--vision/--mmproj, --no-think, --tags)
     tuieval scan                                  # GGUFs in model_dirs that aren't registered yet
     tuieval list                                  # the models tuieval knows; hidden ones listed separately
@@ -53,6 +54,7 @@ class Printer:
     def __init__(self):
         self.lock = threading.Lock()
         self.mode = None
+        self.side_by_side = False   # several models at a time: one line per answer, no live streams
 
     def __call__(self, kind, **d):
         with self.lock:
@@ -73,6 +75,9 @@ class Printer:
     def on_server_stall(self, label, message):
         say(f"STALL: {message}", RED)
 
+    def on_model_waiting(self, label, message):
+        say(message, YELLOW)
+
     def on_server_retry(self, label, message):
         say(message, RED)
 
@@ -97,11 +102,15 @@ class Printer:
             + (f" ({earlier_s / 60:.0f} min over {sittings} earlier sitting(s))" if sittings else ""))
 
     def on_request_started(self, job, test, test_id, has_image, repeat, difficulty="unrated"):
+        if self.side_by_side:
+            return
         tag = {"easy": GREEN, "medium": "\033[33m", "hard": RED}.get(difficulty, DIM) + f"[{difficulty}]" + RESET
         print(f"\n{BOLD}{CYAN}━━ {job.key} [{job.done + 1}/{job.total}]{RESET} {tag} {BOLD}{CYAN}{test}{RESET}", flush=True)
         self.mode = None
 
-    def on_delta(self, stream, text):
+    def on_delta(self, stream, text, label=None):
+        if self.side_by_side:
+            return
         if self.mode != stream:
             sys.stdout.write(f"{DIM}[thinking] " if stream == "reasoning" else f"{RESET}\n{BOLD}[answer]{RESET} ")
             self.mode = stream
@@ -114,8 +123,10 @@ class Printer:
         calls = f" tool={r['tool_calls'][0]['name']}" if r.get("tool_calls") else ""
         speed = f"{r['gen_tps']:.0f} tok/s, " if r.get("gen_tps") else ""
         ttft = f"TTFT {r['ttft_s']:.1f}s, " if r.get("ttft_s") is not None else ""
-        print(f"{RESET}\n{mark} {r['description']}{RESET}{calls} {DIM}({r.get('completion_tokens')} tokens, "
-              f"{ttft}{speed}{r['total_s']:.1f}s) {r['reason'][:120]}{RESET}", flush=True)
+        where = f"{CYAN}{job.key} [{job.done}/{job.total}]{RESET} " if self.side_by_side else ""
+        print(f"{RESET}{'' if self.side_by_side else chr(10)}{where}{mark} {r['description']}{RESET}{calls} "
+              f"{DIM}({r.get('completion_tokens')} tokens, {ttft}{speed}{r['total_s']:.1f}s) {r['reason'][:120]}{RESET}",
+              flush=True)
 
     def on_job_done(self, job):
         color = GREEN if job.status == "done" else RED if job.status == "failed" else ""
@@ -683,6 +694,8 @@ def main(argv=None):
     p.add_argument("--tier", choices=engine.TIERS, default="screen",
                    help="smoke: 3 tests; screen (default): a spread sample; certify: everything, with repeats")
     p.add_argument("--smoke", action="store_const", const="smoke", dest="tier", help="same as --tier smoke")
+    p.add_argument("--parallel", type=int, metavar="N",
+                   help="models served at a time (default: parallel_models for this machine in models.toml, else 1)")
     p.add_argument("--dry-run", action="store_true", help="print the plan and commands without running")
     p.add_argument("--brief", action="store_true", help="end with one line per model instead of the scorecards")
     p.add_argument("--models", default=None, help="default: the workspace's models.toml")
@@ -691,8 +704,12 @@ def main(argv=None):
     p.add_argument("--log-dir", help="server and request logs (default: logs/)")
     args = p.parse_args(argv)
 
-    e = engine.Engine(Printer(), models_path=args.models, packs_dir=args.packs_dir,
+    printer = Printer()
+    e = engine.Engine(printer, models_path=args.models, packs_dir=args.packs_dir,
                       results_dir=args.results_dir, log_dir=args.log_dir)
+    if args.parallel is not None and args.parallel < 1:
+        sys.exit("--parallel must be 1 or more")
+    parallel = args.parallel or e.parallel_models()
     models = e.cfg["models"]
     labels = [m["label"] for m in models if not m.get("remote")]   # hosted APIs cost money: only by name
     if args.only:
@@ -757,7 +774,12 @@ def main(argv=None):
 
     signal.signal(signal.SIGINT, stop)
     signal.signal(signal.SIGTERM, stop)
-    worker = threading.Thread(target=e.run, args=(jobs,))
+    side_by_side = min(parallel, len({j.label for j in jobs if j.status == "waiting"}))
+    if side_by_side > 1:
+        printer.side_by_side = True
+        say(f"up to {side_by_side} models at a time: one line per answer; each answer notes the models "
+            "it ran alongside, since they share this machine's speed")
+    worker = threading.Thread(target=e.run, args=(jobs, parallel))
     worker.start()
     while worker.is_alive():
         worker.join(0.5)

tuieval/tui.py

@@ -290,8 +290,11 @@ HELP = {
 2. Pick a [b]tier[/b]: Smoke first to check the setup, then Screen, then Certify for finalists.
 3. Press [b]s[/b]. The line above the buttons says how many answers that is and roughly how long.
 
+[b]Models at a time[/b]: how many of the ticked models run side by side (blank: this machine's
+parallel_models in models.toml, default 1). Answers note the models they ran alongside.
+
 [b]Keys[/b]
-  s  start (or queue it behind a run that's going)     r  results
+  s  start (with a run going: add to it, or queue it)   r  results
   a  add a model (a GGUF, a model id, openrouter:<id>)  m  scan model folders for new GGUFs
   e  pick which tests of the highlighted pack run       p  save or load a selection (preset)
   t  tune the ticked models' speed flags here (after   x  hide or unhide a model
@@ -307,8 +310,9 @@ with the model's reasoning and answer as they stream, then recent results with t
 
 [b]Keys[/b]
   k  skip the current model          c  cancel everything (finished answers are kept; it resumes next time)
-  f  pause or resume auto-scroll     r  results     n  set up a new run (this one keeps going)
-  w  runs and queue                  esc  back to Setup (the run keeps going)""",
+  f  pause or resume auto-scroll     r  results     n  more evals: add them to this run or queue them
+  w  runs and queue                  esc  back to Setup (the run keeps going)
+  v  with several models at a time: stream the next one (k skips the one streaming)""",
     "ResultsScreen": """[b]Results[/b]
 
 Start with [b]Production readiness[/b]: one verdict per model and use case, and what's missing when
@@ -435,6 +439,7 @@ class AnswerScreen(ModalScreen):
                  f"{rec['gen_tps']:.0f} tok/s" if rec.get("gen_tps") else "",
                  f"{rec['total_s']:.0f}s total" if rec.get("total_s") is not None else "",
                  f"on {rec['machine']}" if rec.get("machine") else "",
+                 f"[yellow]ran alongside {', '.join(rec['ran_alongside'])}[/yellow]" if rec.get("ran_alongside") else "",
                  f"seed {rec['seed']}" if "seed" in rec else ""]
         self.query_one("#answer-head", Static).update(
             f"{mark}  {diff_badge(rec.get('difficulty', 'unrated'))}  [b]{rich_escape(rec.get('description', rec['test']))}[/b]"
@@ -544,6 +549,8 @@ class SetupScreen(Screen):
         with Horizontal(id="options"):
             yield Label("Repeats")
             yield Input("", id="repeat", type="integer", max_length=2, placeholder="1")
+            yield Label("Models at a time")
+            yield Input("", id="parallel", type="integer", max_length=1, placeholder="1")
             yield Checkbox("Rerun existing results", id="force")
         yield Static(id="estimate")
         with Horizontal(id="buttons"):
@@ -761,6 +768,13 @@ class SetupScreen(Screen):
                 list(self.query_one("#suites", SelectionList).selected), repeat,
                 self.tier(), self.query_one("#force", Checkbox).value)
 
+    def parallel(self):
+        """Models at a time for the next run: the field, or blank for this machine's parallel_models."""
+        try:
+            return max(1, int(self.query_one("#parallel", Input).value)) or None
+        except ValueError:
+            return None
+
     def current_picks(self, suites=None):
         """{pack: picked test ids} for packs that still have those tests (and, given suites, are ticked)."""
         e = self.app.engine
@@ -803,6 +817,7 @@ class SetupScreen(Screen):
         e = self.app.engine
         defaults = sorted({e.default_repeat(e.packs[p], tier) for p in suites}) or [1]
         self.query_one("#repeat", Input).placeholder = str(defaults[0]) if len(defaults) == 1 else "pack"
+        self.query_one("#parallel", Input).placeholder = str(e.parallel_models())
         if not e.packs:
             est.update("[yellow]No packs yet.[/yellow] [dim]Quit, run tuieval new-pack <name>, "
                        "and open tuieval again.[/dim]")
@@ -852,13 +867,19 @@ class SetupScreen(Screen):
         resumed = [j for j in run if "done earlier" in j.note]
         if resumed:
             text += f"   [yellow]{len(resumed)} pack(s) continue from earlier results[/yellow]"
+        at_once = min(self.parallel() or e.parallel_models(), len({j.label for j in run}))
+        if at_once > 1:
+            text += (f"\n[cyan]Up to {at_once} models at a time[/cyan][dim] (the time above assumes one at a "
+                     "time). Answers note the models they ran alongside, since those share the machine's "
+                     "speed.[/dim]")
         est.update(text + self.queue_hint())
 
     def queue_hint(self):
         """Says what Start will do when something is already running, here or in another window."""
         app = self.app
         if app.is_busy():
-            return (f"\n[cyan]{app.busy_text()}: Start adds this to the queue as #{len(app.queue) + 1} "
+            add = "add this to the run going or " if isinstance(app.active_session, RunScreen) else ""
+            return (f"\n[cyan]{app.busy_text()}: Start lets you {add}queue it as #{len(app.queue) + 1} "
                     "(w shows the queue).[/cyan]")
         holder = app.engine.machine_holder()
         if holder:
@@ -879,6 +900,8 @@ class SetupScreen(Screen):
             if name in e.packs:
                 packs.select(name)
         self.query_one("#repeat", Input).value = str(sel["repeat"]) if sel.get("repeat") else ""
+        if "parallel" in sel:
+            self.query_one("#parallel", Input).value = str(sel["parallel"]) if sel["parallel"] else ""
         tier = sel.get("tier") or ("smoke" if sel.get("smoke") else None)
         if tier in engine.TIERS:
             self.query_one(f"#tier-{tier}", RadioButton).value = True
@@ -917,7 +940,8 @@ class SetupScreen(Screen):
             self.notify("Select at least one eval pack and one model.", severity="warning")
             return
         picks = self.current_picks(suites)
-        sel = {"models": labels, "packs": suites, "repeat": repeat, "tier": tier, "force": force, "tests": picks}
+        sel = {"models": labels, "packs": suites, "repeat": repeat, "tier": tier, "force": force, "tests": picks,
+               "parallel": self.parallel()}
         jobs = self.app.engine.plan(labels, suites, repeat, tier, force, picks)
         rerun = False
         if not any(j.status == "waiting" for j in jobs):
@@ -1441,8 +1465,8 @@ class QueuedRun:
     def line(self):
         return f"{self.title}  [dim]~{fmt_secs(self.est)} · queued {time.strftime('%H:%M', time.localtime(self.queued))}[/dim]"
 
-    def make(self, eng):
-        """(RunScreen, None), or (None, why it can't start) when its turn comes."""
+    def plan(self, eng):
+        """(jobs, None), or (None, why it can't run) now."""
         sel = self.sel
         jobs = eng.plan(sel["models"], sel["packs"], sel["repeat"], sel["tier"], sel["force"] or self.rerun,
                         self.tests(eng))
@@ -1453,7 +1477,12 @@ class QueuedRun:
         if new:
             return None, (f"it would now start over {', '.join(new)}, which wasn't confirmed when it was queued. "
                           "Start it again from Setup to see why")
-        return RunScreen(jobs), None
+        return jobs, None
+
+    def make(self, eng):
+        """(RunScreen, None), or (None, why it can't start) when its turn comes."""
+        jobs, problem = self.plan(eng)
+        return (RunScreen(jobs, self.sel.get("parallel")), None) if jobs else (None, problem)
 
 
 class QueuedTune:
@@ -1783,7 +1812,8 @@ class RunScreen(Screen):
         Binding("k", "skip_model", "Skip model", priority=True),
         Binding("c", "cancel_all", "Cancel all", priority=True),
         Binding("f", "toggle_follow", "Pause scroll", priority=True),
-        Binding("n", "new_run", "New run", priority=True),
+        Binding("v", "watch_next", "Next model", priority=True),
+        Binding("n", "new_run", "More evals", priority=True),
         Binding("r", "results", "Results", priority=True),
         Binding("w", "runs", "Runs", priority=True),
         Binding("escape", "back", "Setup", priority=True),
@@ -1791,11 +1821,13 @@ class RunScreen(Screen):
     ]
     kind = "Run"
     LIVE_ONLY = {"skip_model", "cancel_all", "toggle_follow"}
-    DONE_ONLY = {"new_run"}
+    DONE_ONLY = set()
 
-    def __init__(self, jobs):
+    def __init__(self, jobs, parallel=None):
         super().__init__()
         self.jobs = jobs
+        self.parallel = parallel      # models at a time (None: this machine's parallel_models)
+        self.watch = None             # the model whose answers stream (several run at a time)
         self.running = True
         self.follow = True
         self.buf = {"reasoning": [], "answer": []}
@@ -1839,12 +1871,24 @@ class RunScreen(Screen):
         self.notify("\n".join(lines), title=title, severity="error", timeout=30)
 
     def check_action(self, action, parameters):
+        if action == "watch_next":
+            return self.running and self.side_by_side
         if action in self.LIVE_ONLY:
             return self.running
         if action in self.DONE_ONLY:
             return not self.running
         return True
 
+    @property
+    def side_by_side(self):
+        """Several models run at a time in this run."""
+        return min(self.parallel or self.app.engine.parallel_models(),
+                   len({j.label for j in self.jobs if j.status != "skipped"})) > 1
+
+    def active(self):
+        """Models being loaded or run now, in queue order."""
+        return list(dict.fromkeys(j.label for j in self.jobs if j.status in ("loading", "running")))
+
     def counts(self):
         return (sum(j.done for j in self.jobs), sum(j.passed for j in self.jobs), sum(j.failed for j in self.jobs))
 
@@ -1908,14 +1952,18 @@ class RunScreen(Screen):
         self.query_one("#bar", ProgressBar).update(total=max(self.total, 1), progress=0)
         self.set_interval(0.1, self.flush_streams)
         self.set_interval(1.0, self.tick)
-        self.app.start_run(self.jobs, self.on_engine_event)
+        self.app.start_run(self.jobs, self.on_engine_event, self.parallel)
         self.tick()
 
     # -- events from the engine thread
     def on_engine_event(self, kind, **d):
         if self.app.closing:
             return  # the app is shutting down; the UI thread may be waiting on the engine
+        if kind in ("request_started", "model_loading") and self.watch not in self.active():
+            self.watch = d["job"].label if "job" in d else d["label"]   # follow the next model streaming
         if kind == "delta":  # very frequent: buffer, flushed every 100 ms on the UI thread
+            if d.get("label", self.watch) != self.watch:
+                return       # another model running alongside: v switches to it
             with self.buf_lock:
                 self.buf[d["stream"]].append(d["text"])
             return
@@ -1936,8 +1984,15 @@ class RunScreen(Screen):
     def event(self, text):
         self.query_one("#events", Log).write_line(time.strftime("%H:%M:%S ") + text)
 
+    def watched(self, kind, d):
+        """Whether an event belongs to the streaming model (always, with one model at a time)."""
+        label = d["job"].label if "job" in d else d.get("label")
+        return label is None or label == self.watch or not self.side_by_side
+
     def handle(self, kind, d):
-        if kind == "request_started":
+        if kind == "request_started" and not self.watched(kind, d):
+            pass         # another model running alongside: its rows still show in Recent results
+        elif kind == "request_started":
             self.flush_streams()
             self.query_one("#reasoning", StreamView).clear()
             self.query_one("#answer", StreamView).clear()
@@ -1948,7 +2003,8 @@ class RunScreen(Screen):
                             f"{'  [magenta]🖼 image[/magenta]' if d['has_image'] else ''}\n"
                             f"{diff_badge(d.get('difficulty', 'unrated'))}  {d['test']}")
         elif kind == "request_done":
-            self.flush_streams()
+            if self.watched(kind, d):
+                self.flush_streams()
             job, r = d["job"], d["record"]
             mark = "[green]✓[/green]" if r["pass"] else "[red]✗[/red]"
             recent = self.query_one("#recent", DataTable)
@@ -1974,7 +2030,18 @@ class RunScreen(Screen):
                 recent.move_cursor(row=recent.row_count - 1)
             if job:
                 self.update_job(job)
-            self.req_started = None
+            if self.watched(kind, d):
+                self.req_started = None
+            self.update_overall()
+        elif kind == "jobs_added":
+            q = self.query_one("#queue", DataTable)
+            for j in d["jobs"]:
+                if j.key not in q.rows:
+                    q.add_row(j.label, j.pack.label, "", "", "", "", "", "", key=j.key)
+                self.update_job(j)
+                self.total += j.total
+            self.query_one("#bar", ProgressBar).update(total=max(self.total, 1))
+            self.event(f"added to this run: {', '.join(j.key for j in d['jobs'])}")
             self.update_overall()
         elif kind in ("job_started", "job_update", "job_done"):
             job = d["job"]
@@ -2008,15 +2075,18 @@ class RunScreen(Screen):
             self.current = "The other window finished; starting…"
             self.event("the machine is free; starting")
         elif kind == "model_loading":
-            self.current = f"Loading [b]{d['label']}[/b]…"
             self.event(f"{d['label']}: starting server")
             if d.get("log"):
                 self.server_logs[d["label"]] = d["log"]
-            self.query_one("#serverlog", Log).write_line(f"$ {' '.join(d['command'])}")
-            self.query_one("#reasoning", StreamView).clear()
-            self.query_one("#answer", StreamView).clear()
+            self.query_one("#serverlog", Log).write_line(self.log_prefix(d) + f"$ {' '.join(d['command'])}")
+            if self.watched(kind, d):
+                self.current = f"Loading [b]{d['label']}[/b]…"
+                self.query_one("#reasoning", StreamView).clear()
+                self.query_one("#answer", StreamView).clear()
+        elif kind == "model_waiting":
+            self.event(f"[yellow]{d['message']}[/yellow]")
         elif kind == "server_log":
-            self.query_one("#serverlog", Log).write_line(d["line"])
+            self.query_one("#serverlog", Log).write_line(self.log_prefix(d) + d["line"])
         elif kind == "server_stall":
             self.event(f"[bold red]STALL[/bold red] {d['message']}")
             self.notify(d["message"], severity="error", timeout=30)
@@ -2029,7 +2099,8 @@ class RunScreen(Screen):
         elif kind == "model_ready":
             loaded = f" in {d['load_s']:.0f}s" if d.get("load_s") else ""
             self.event(f"{d['label']}: ready as {d['ids']}{loaded}")
-            self.current = f"[b]{d['label']}[/b] ready{loaded}, starting evals…"
+            if self.watched(kind, d):
+                self.current = f"[b]{d['label']}[/b] ready{loaded}, starting evals…"
         elif kind == "resource":
             self.mem[d["label"]] = d["rss_mb"]
         elif kind == "server_retry":
@@ -2082,6 +2153,9 @@ class RunScreen(Screen):
             self.app.session_finished(self)
         self.refresh_current()
 
+    def log_prefix(self, d):
+        return f"[{d['label']}] " if self.side_by_side and d.get("label") else ""
+
     def update_job(self, j):
         q = self.query_one("#queue", DataTable)
         style = STATUS_STYLE.get(j.status, "")
@@ -2115,7 +2189,9 @@ class RunScreen(Screen):
             speed += f" · {sorted(tps)[len(tps) // 2]:.0f} tok/s"
         if ttfts:
             speed += f" · TTFT {sorted(ttfts)[len(ttfts) // 2]:.1f}s"
-        mem = max(self.mem.values()) if self.mem else 0
+        active = self.active()
+        mem = (sum(self.mem.get(l, 0) for l in active) if self.side_by_side and active   # models side by side add up
+               else max(self.mem.values()) if self.mem else 0)
         if mem:
             speed += f" · mem {mem / 1024:.1f} GB" if mem >= 1024 else f" · mem {mem:.0f} MB"
         self.query_one("#overall", Static).update(
@@ -2132,6 +2208,12 @@ class RunScreen(Screen):
             if self.req_first and now - self.req_first > 1:
                 rate = f" · ~{self.req_tokens / (now - self.req_first):.0f} tok/s"
             extra = f"   [dim]{fmt_secs(now - self.req_started)} · {ttft}~{self.req_tokens} tokens{rate}[/dim]"
+        others = [l for l in self.active() if l != self.watch] if self.running and self.side_by_side else []
+        if others:
+            def where(label):
+                j = next((j for j in self.jobs if j.label == label and j.status in ("loading", "running")), None)
+                return f"{label} ({j.pack.label} {j.done}/{j.total})" if j and j.status == "running" else f"{label} (loading)"
+            extra += f"\n[dim]Also running: {' · '.join(map(where, others))} · v streams the next one[/dim]"
         self.query_one("#current", Static).update(self.current + extra)
 
     def tick(self):
@@ -2144,8 +2226,28 @@ class RunScreen(Screen):
     # -- actions
     def action_skip_model(self):
         if self.running:
-            self.app.engine.skip_model()
-            self.notify("Skipping the current model…")
+            if self.side_by_side and self.watch:
+                self.app.engine.skip_model(self.watch)
+                self.notify(f"Skipping {self.watch}; the others keep going…")
+            else:
+                self.app.engine.skip_model()
+                self.notify("Skipping the current model…")
+
+    def action_watch_next(self):
+        """Stream the next model running alongside (from its next answer)."""
+        active = self.active()
+        if len(active) < 2:
+            self.notify("Only one model is running right now.")
+            return
+        self.watch = active[(active.index(self.watch) + 1) % len(active)] if self.watch in active else active[0]
+        with self.buf_lock:
+            for v in self.buf.values():
+                v.clear()
+        self.query_one("#reasoning", StreamView).clear()
+        self.query_one("#answer", StreamView).clear()
+        self.req_started = None
+        self.current = f"Streaming [b]{self.watch}[/b] from its next answer…"
+        self.refresh_current()
 
     def action_cancel_all(self):
         if not self.running:
@@ -2176,6 +2278,9 @@ class RunScreen(Screen):
 
     @on(Button.Pressed, "#done-new")
     def action_new_run(self):
+        if self.running:
+            self.notify("Pick models and packs, then Start: add them to this run or queue them after it.",
+                        timeout=8)
         self.app.back_to_setup()  # Setup still holds the last selection
         self.app.screen.query_one("#models").focus()
 
@@ -2548,6 +2653,11 @@ class ResultsScreen(Screen):
                      f"failed; both passed {h['both']}, both failed {h['neither']}. Fewer tokens: {a} on "
                      f"{h['fewer_a']}, {b} on {h['fewer_b']}" + (f" (median {a}/{b} ratio {h['ratio']:.2f}×)"
                                                                if h["ratio"] else "") + ".")
+        shared_speed = sorted({c_m for t in shared for c_m, c in t["cells"].items() if c_m in models
+                               and any(r.get("alongside") for r in c["rows"])})
+        if shared_speed:
+            line += (f"\n[yellow]Some answers of {', '.join(shared_speed)} ran alongside other models, which "
+                     "shared the machine's speed: compare their times with care.[/yellow]")
         title.update(line + "  [dim]Fewest tokens / Fastest = questions won; ties count for both. Total tokens / "
                      "Total time add up each question's median. Avg tok/s = all tokens over all the time they took "
                      "(cached prompts left out).[/dim]")
@@ -2676,6 +2786,7 @@ class EvalsApp(App):
     #options { height: 3; padding: 0 1; align-vertical: middle; }
     #options Label { padding: 1 1 0 0; }
     #options Input { width: 12; }
+    #options #parallel { width: 8; }
     #estimate { padding: 0 2; height: auto; min-height: 2; }
     #buttons { height: 3; padding: 0 1; }
     #buttons Button { margin-right: 2; }
@@ -2831,6 +2942,40 @@ class EvalsApp(App):
                 self.notify("Everything selected had results; running it again.")
             self.add_session(screen)
             return
+        run = self.active_session
+        if item.kind == "Run" and isinstance(run, RunScreen) and run.running \
+                and item.sel["tier"] != run.jobs[0].tier:
+            self.notify(f"The run going is {run.jobs[0].tier.title()} and this is {item.sel['tier'].title()}, "
+                        "so it can't join it.", timeout=10)
+        elif item.kind == "Run" and isinstance(run, RunScreen) and run.running:
+            def picked(choice):
+                if choice == "add":
+                    self.add_to_run(run, item)
+                elif choice == "queue":
+                    self.enqueue(item)
+            what = f"{', '.join(item.sel['models'][:4])}{' …' if len(item.sel['models']) > 4 else ''} · " \
+                   f"{', '.join(item.sel['packs'][:4])}{' …' if len(item.sel['packs']) > 4 else ''}"
+            self.push_screen(ChoiceScreen(
+                f"A run is going: {run.live_text()}.\n\nAdd {what} to it? Its models run after the ones "
+                "already in it.\nOr queue it as a separate run that starts when this one ends.",
+                [("add", "Add to this run", "success"), ("queue", f"Queue it as #{len(self.queue) + 1}", "primary"),
+                 ("keep", "Cancel", "default")]), picked)
+            return
+        self.enqueue(item)
+
+    def add_to_run(self, run, item):
+        jobs, problem = item.plan(self.engine)
+        added, why = self.engine.add_jobs(jobs) if jobs else ([], problem)
+        if not added:
+            self.notify(f"Not added: {why}. Queued it instead.", severity="warning", timeout=10)
+            self.enqueue(item)
+            return
+        run.handle("jobs_added", {"jobs": added})
+        models = sorted({j.label for j in added})
+        self.notify(f"Added {len(added)} pack run(s) to the run going: {', '.join(models)}. w shows it.", timeout=10)
+        self.refresh_live()
+
+    def enqueue(self, item):
         ahead = self.busy_text()
         self.queue.append(item)
         self.notify(f"Queued #{len(self.queue)}: {item.title}. It starts by itself after: {ahead}. "
@@ -2965,10 +3110,10 @@ class EvalsApp(App):
         self.worker = threading.Thread(target=work, name="tune")
         self.worker.start()
 
-    def start_run(self, jobs, on_event):
+    def start_run(self, jobs, on_event, parallel=None):
         self.engine.on_event = on_event
         # A plain (non-daemon) thread, so quitting can wait for servers to be stopped.
-        self.worker = threading.Thread(target=self.engine.run, args=(jobs,), name="engine")
+        self.worker = threading.Thread(target=self.engine.run, args=(jobs, parallel), name="engine")
         self.worker.start()
 
     def on_unmount(self):

{tuieval-0.2.0.dev25.dist-info → tuieval-0.2.0.dev32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev25
+Version: 0.2.0.dev32
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -137,7 +137,7 @@ See **[docs/models.md](docs/models.md)** for servers, OpenRouter endpoint pinnin
 
 1. **Pick packs and models** (space ticks; type to filter models by label or tag). Each model shows a short code per use case, e.g. `C✓ S?` (✓ pass, ✗ fail, ? inconclusive; grey = from earlier results). To run only some tests of a pack, highlight it and press `e`. Runs of different tests add up, and the pack can PASS once all its tests have run.
 2. **Pick a tier** (Smoke to check setup, Screen, Certify) and **press `s`**. The line above the buttons shows how many answers that is and roughly how long it will take. For each model, tuieval starts its server (or uses a running one), checks the right model is loaded, runs every selected pack, then stops it.
-3. **Watch the run:** progress with ETA, live score, tok/s, TTFT and memory, the current test with reasoning and answer side by side, recent results with the grader's reason. `k` skips a model, `c` cancels (finished work is kept and resumes next time). Runs started while one is going are queued.
+3. **Watch the run:** progress with ETA, live score, tok/s, TTFT and memory, the current test with reasoning and answer side by side, recent results with the grader's reason. `k` skips a model, `c` cancels (finished work is kept and resumes next time). `n` sets up more evals while it runs: add them to this run (same tier) or queue them as the next run.
 4. **Press `r` for results:** production readiness, verdict history, scorecard, speed & tokens (★ marks models nothing beats on both accuracy and time), per question, is the difference real?, tests that separate models, by difficulty, failures, and test quality.
 
 Other keys: `t` tunes the ticked models' speed flags, `a` adds a model, `m` scans for GGUFs, `p` saves or loads a preset, `x` hides a model. `?` explains any screen.

{tuieval-0.2.0.dev25.dist-info → tuieval-0.2.0.dev32.dist-info}/RECORD

@@ -1,22 +1,22 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=jkFBO1NDLAWZMcQgvtkutxfpSaQvK83P67EheKl8tZI,535
+tuieval/_version.py,sha256=YtBX5IRrDzkDypUA_3LD09E8ezHBpeqSVHqjkEVNvp4,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=7joEA07W2we-kcg4VXxfCSXazj7dWBCKlQlSXpXQfTc,4390
 tuieval/client.py,sha256=Vui7ucVv_GiKObGoVS_EWSZFUKPou9PWmutqMTrahXk,9313
-tuieval/compare.py,sha256=b-rVcMwq0Tos3oNKcpUP1HXULKhAQNAS4rlXYSY7t6k,25460
+tuieval/compare.py,sha256=SHN_o_nlppFcHMBqBZRITIkeQDPaR_JxhUZItQLMBwo,26270
 tuieval/doctor.py,sha256=y1C_ggjd8oikXTrovgTBf26wjThP8EoVkfTVeO_n5dQ,6070
-tuieval/engine.py,sha256=aRZn-PgB8e4DCLJNPsMhIsF6Kk-CHB_CIsP2ECRizy0,103183
+tuieval/engine.py,sha256=xTDlcU2nPGcSvv6a45gLTsMU7p7ZvNxmzrkGPMe9vfQ,113950
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/machines.py,sha256=ImEihNj8QlBRdCHLED4QIdjJUvjVtZgo6DiVU5h6Zzs,13415
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=nGkbMj0qE4G4RSgIK1lradOFD_8weITk0w54TRNxqI0,42426
+tuieval/run_evals.py,sha256=ZGx4XcZ15qD3vIg15xCfdX4SxTDHXBp_GYoiGexchdw,43668
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=aG-KMsD_8ma27fsHvw01S7V_g10pzJ-nrWi5pq28DA8,151175
+tuieval/tui.py,sha256=thjGLDQCZ5YkFmtZ-xB3914ez7I5S2wePwZlFbcqggg,159589
 tuieval/tune.py,sha256=jXwYuSjfdqbLCE3WonP2xaUHwk_FnhI67twB1BaHggI,38992
 tuieval/verdict.py,sha256=Muo1a05mCv781MlEJsXwDsGaKbRi0JS4Lv8HdvK_Aco,19054
 tuieval/watch_proxy.py,sha256=z916NRhEur04a9RnLm6ZpIEY1SnnUGPiHYbmlD9ktQ0,11035
@@ -45,8 +45,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev25.dist-info/METADATA,sha256=dsC0AUbQRdPTzQzc7h6KGTmhkYpZswqme7BSCDSdYUo,13227
-tuieval-0.2.0.dev25.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev25.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev25.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev25.dist-info/RECORD,,
+tuieval-0.2.0.dev32.dist-info/METADATA,sha256=ZIxzM44ul8qTlthu0MI02bX8jD_09tVkO_WI3cjbah4,13285
+tuieval-0.2.0.dev32.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev32.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev32.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev32.dist-info/RECORD,,