tuieval 0.2.0.dev23__py3-none-any.whl → 0.2.0.dev28__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev23'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev23')
+__version__ = version = '0.2.0.dev28'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev28')
 
 __commit_id__ = commit_id = None

tuieval/compare.py

@@ -37,7 +37,8 @@ def _load_native(data):
             "settings": data.get("settings", {}), "load_s": run.get("load_s"),
             "peak_rss_mb": run.get("peak_rss_mb"), "pack": pack["name"], "pack_fp": pack["fingerprint"],
             # hosted (OpenRouter) models: the provider endpoint and its declared quantization
-            "endpoint": run.get("endpoint"), "quantization": run.get("quantization")}
+            "endpoint": run.get("endpoint"), "quantization": run.get("quantization"),
+            "loaded_alongside": run.get("loaded_alongside") or []}
     rows = []
     for r in data["results"]:
         rows.append({
@@ -54,6 +55,8 @@ def _load_native(data):
             "severity": r.get("severity"), "group": r.get("group"), "repeat": r.get("repeat", 0),
             "difficulty": r.get("difficulty", "unrated"),
             "machine": r.get("machine") or run.get("machine"), "prompt_tokens": r.get("prompt_tokens"),
+            # other models served at the same time (parallel_models): its speed was shared with them
+            "alongside": r.get("ran_alongside") or [],
         })
     return rows, info
 
@@ -210,6 +213,13 @@ def timing(r, machine=None, speeds=None):
     return None, None, None, None, None
 
 
+def alongside_note(rows):
+    """' · n/N ran alongside others' when some answers ran while other models were served (their
+    speed was shared with them), else ''."""
+    shared = sum(bool(r.get("alongside")) for r in rows)
+    return f" · {shared}/{len(rows)} ran alongside others" if shared else ""
+
+
 def speed(rows, infos, machine=None, speeds=None):
     """(header, table, frontier): speed and token use per model.
 
@@ -227,7 +237,9 @@ def speed(rows, infos, machine=None, speeds=None):
     load = {}
     for i in infos:
         cur = load.setdefault(i["label"], {"load_s": None, "peak_rss_mb": None, "tags": i["tags"],
-                                           "think": i["settings"].get("enable_thinking"), "served": set()})
+                                           "think": i["settings"].get("enable_thinking"), "served": set(),
+                                           "loaded_alongside": False})
+        cur["loaded_alongside"] |= bool(i.get("loaded_alongside"))
         if i.get("endpoint"):
             q = i.get("quantization")
             cur["served"].add(f"{'undeclared' if q in (None, 'unknown') else q} via {i['endpoint']}")
@@ -247,6 +259,7 @@ def speed(rows, infos, machine=None, speeds=None):
         here = machine is None or machine in where   # load time and memory only where it ran
         if machine is None:
             kind = "measured on " + (", ".join(where) or "?")
+        kind += alongside_note(v)
         table.append([
             "★" if m in frontier else "", m, "; ".join(sorted(info.get("served") or [])) or "-",
             ",".join(info.get("tags") or []),
@@ -256,7 +269,8 @@ def speed(rows, infos, machine=None, speeds=None):
             _fmt(median(x[3] for x in t), ".0f"), _fmt(median(r["tokens"] for r in v), ".0f"),
             _fmt(median(r["reasoning_tokens"] for r in v), ".0f"), _fmt(median(r["answer_tokens"] for r in v), ".0f"),
             f"{sum(r['tokens'] for r in v) / passes:,.0f}" if passes else "-",
-            _fmt(info.get("load_s") if here else None, ".0f"),
+            _fmt(info.get("load_s") if here else None, ".0f")
+            + (" (alongside others)" if here and info.get("load_s") and info.get("loaded_alongside") else ""),
             _fmt(info["peak_rss_mb"] / 1024 if info.get("peak_rss_mb") and here else None, ".1f"),
         ])
     return header, table, frontier

tuieval/engine.py

@@ -229,6 +229,7 @@ class Serving:
     identity: dict             # output-affecting settings, part of the results fingerprint
     profile: dict | None = None
     mtp_layers: int = 0
+    need_gb: float | None = None   # estimated GPU memory at ctx (from the fit check); None when unknown
 
 
 def infer_server(cfg, model):
@@ -664,6 +665,20 @@ def holder_text(holder):
             "One eval or tune per machine: this starts when it ends")
 
 
+class Lane:
+    """One model being run: its own request stream, stall reason and skip switch, so models run
+    side by side (parallel_models) stop independently. Runs and tunes in one lane use the engine's
+    main lane."""
+    def __init__(self, label=None):
+        self.label = label
+        self.stream = None
+        self.stall = None         # set by the stall watcher: why this model's server stalled
+        self.skip = threading.Event()
+        self.procs = []
+        self.port = None
+        self.seen = set()         # other models served during this model's current load or request
+
+
 class Engine:
     def __init__(self, on_event=None, root=None, models_path=None, packs_dir=None,
                  results_dir=None, log_dir=None):
@@ -677,11 +692,12 @@ class Engine:
         self.pack_errors = []
         self.packs = packs_mod.load_packs(self.packs_dir, self.pack_errors)
         self._cancel = threading.Event()
-        self._skip = threading.Event()
-        self._stall = None        # set by the stall watcher: why the current server stalled
+        self._main_lane = Lane()
+        self._here = threading.local()   # .lane: the lane this thread runs
+        self._lanes = {}          # label -> Lane of each model being served now
+        self._lanes_lock = threading.Lock()
         self._procs = []
         self._stop_lock = threading.Lock()
-        self._stream = None
         self._header_cache = {}
         self._serving = {}
         self.tuning_dir = os.path.join(root, "tuning")
@@ -690,6 +706,34 @@ class Engine:
         self._catalogues = {}
         self._add_remote_models()
 
+    # ---- the lane this thread runs (see Lane)
+    def _lane(self):
+        return getattr(self._here, "lane", None) or self._main_lane
+
+    def _all_lanes(self):
+        with self._lanes_lock:
+            return [self._main_lane, *self._lanes.values()]
+
+    @property
+    def _stream(self):
+        return self._lane().stream
+
+    @_stream.setter
+    def _stream(self, value):
+        self._lane().stream = value
+
+    @property
+    def _stall(self):
+        return self._lane().stall
+
+    @_stall.setter
+    def _stall(self, value):
+        self._lane().stall = value
+
+    @property
+    def _skip(self):
+        return self._lane().skip
+
     def reload(self):
         self._serving = {}
         self.cfg = load_config(self.models_path)
@@ -1150,6 +1194,30 @@ class Engine:
         """models.toml [machines.<id>] (optional): memory_headroom_gb, name."""
         return self.cfg.get("machines", {}).get(machine_id or self.machine().id, {})
 
+    def parallel_models(self):
+        """How many models a run serves at a time on this machine: models.toml [machines.<id>]
+        parallel_models (default 1)."""
+        try:
+            return max(1, int(self.machine_settings().get("parallel_models", 1)))
+        except (TypeError, ValueError):
+            return 1
+
+    def memory_need_gb(self, m):
+        """GPU memory a model takes while served here: 0 for one we don't start (a hosted API or a
+        server already running), the fit check's estimate for a local GGUF, else None (unknown)."""
+        if not self.cfg["servers"][m["server"]].get("cmd"):
+            return 0.0
+        return self.serving(m).need_gb
+
+    def memory_available_gb(self):
+        """GPU memory models may take here (what the fit check sizes contexts against)."""
+        mc = self.machine()
+        return mc.gpu_limit_gb - self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
+
+    def runs_alone(self, m):
+        """A server whose before_start step may stop other servers never runs next to another model."""
+        return bool(self.cfg["servers"][m["server"]].get("before_start"))
+
     def gpu_residency_gb(self, machine=None):
         """GPU memory this machine's driver keeps resident without churning (see machines.py)."""
         mc = machine or self.machine()
@@ -1180,7 +1248,7 @@ class Engine:
         kv = m.get("kv_type") or server.get("kv_type")
         want = min(x for x in (server.get("max_ctx"), m.get("ctx"), m.get("max_context"), 10**9) if x)
         want = None if want == 10**9 else want
-        ctx, fits, note, mtp, size = want, True, "", 0, None
+        ctx, fits, note, mtp, size, need = want, True, "", 0, None, None
         path = expand(m["model"])
         if server.get("model_is_path") and os.path.isfile(path):
             try:
@@ -1192,7 +1260,7 @@ class Engine:
                     headroom = self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
                     f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=headroom, extra_bytes=extra,
                                      want_ctx=want)
-                    ctx, fits, note = f.max_ctx, f.fits, f.note
+                    ctx, fits, note, need = f.max_ctx, f.fits, f.note, f.need_gb
             except (OSError, ValueError) as e:
                 note = f"couldn't read the GGUF header ({e}); context not checked"
         profile, perf, source = None, [], "n/a"
@@ -1221,7 +1289,7 @@ class Engine:
             # resident), so both are part of what a result is valid for.
             identity["machine"] = mc.id
             identity["perf"] = perf
-        sv = Serving(mc, ctx, kv, fits, note, perf, source, identity, profile, mtp)
+        sv = Serving(mc, ctx, kv, fits, note, perf, source, identity, profile, mtp, need)
         self._serving[key] = sv
         return sv
 
@@ -1289,16 +1357,21 @@ class Engine:
     # ---- control
     def cancel(self):
         self._cancel.set()
-        self._abort()
-
-    def skip_model(self):
-        self._skip.set()
-        self._abort()
-
-    def _abort(self):
-        if self._stream:
-            self._stream.close()
-        for p in list(self._procs):
+        self._abort(self._all_lanes(), list(self._procs))
+
+    def skip_model(self, label=None):
+        """Skip a model being run (label), or every one being run now."""
+        with self._lanes_lock:
+            lanes = [self._lanes[label]] if label in self._lanes else list(self._lanes.values()) or [self._main_lane]
+        for lane in lanes:
+            lane.skip.set()
+        self._abort(lanes, [p for lane in lanes for p in lane.procs] if self._lanes else list(self._procs))
+
+    def _abort(self, lanes, procs):
+        for lane in lanes:
+            if lane.stream:
+                lane.stream.close()
+        for p in procs:
             self._stop(p, grace=10)
 
     def _check(self):
@@ -1322,6 +1395,7 @@ class Engine:
         STALL_WINDOW_S, the GPU driver is churning memory: record why and abort the request."""
         mc = self.machine()
         limit = self.gpu_residency_gb(mc)
+        lane = self._lane()
         samples = []   # (time, user, kernel)
 
         def run():
@@ -1343,15 +1417,15 @@ class Engine:
                     continue
                 share = dk / (du + dk)
                 info["kernel_share_max"] = max(info.get("kernel_share_max") or 0, round(share, 2))
-                if share > self.STALL_KERNEL_SHARE and (du + dk) / dt > self.STALL_MIN_BUSY and not self._stall:
+                if share > self.STALL_KERNEL_SHARE and (du + dk) / dt > self.STALL_MIN_BUSY and not lane.stall:
                     over = f"{alloc:.1f} GB of GPU memory allocated; " if alloc else ""
-                    self._stall = (f"{m['label']} stalled in the GPU driver on {mc.id}: {100 * share:.0f}% of its "
+                    lane.stall = (f"{m['label']} stalled in the GPU driver on {mc.id}: {100 * share:.0f}% of its "
                                    f"CPU time went to the kernel for {self.STALL_WINDOW_S}s while the GPU waited "
                                    f"({over}this machine keeps ~{limit:.0f} GB resident). "
                                    "Lower its GPU memory or retune it.")
-                    self.emit("server_stall", label=m["label"], message=self._stall)
-                    if self._stream:
-                        self._stream.close()
+                    self.emit("server_stall", label=m["label"], message=lane.stall)
+                    if lane.stream:
+                        lane.stream.close()
         threading.Thread(target=run, daemon=True).start()
 
     # ---- processes
@@ -1360,6 +1434,7 @@ class Engine:
         p = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.DEVNULL, stdout=out,
                              stderr=subprocess.STDOUT, start_new_session=True)
         self._procs.append(p)
+        self._lane().procs.append(p)
         return p
 
     def _stop(self, p, grace=30):
@@ -1381,6 +1456,9 @@ class Engine:
                     p.wait()
             if p in self._procs:
                 self._procs.remove(p)
+            for lane in self._all_lanes():
+                if p in lane.procs:
+                    lane.procs.remove(p)
 
     def _tail(self, path, label, stop):
         def run():
@@ -1472,8 +1550,9 @@ class Engine:
         return any(self.cfg["servers"][j.model["server"]].get("cmd") for j in jobs if j.status == "waiting")
 
     # ---- running
-    def run(self, jobs):
-        """Run the jobs (blocking). Jobs of the same model share one server start."""
+    def run(self, jobs, parallel=None):
+        """Run the jobs (blocking). Jobs of the same model share one server start. parallel: how many
+        models are served at a time (default: parallel_models for this machine)."""
         self._cancel.clear()
         self.emit("queue_started", jobs=jobs)
         models = sorted({j.label for j in jobs if j.status == "waiting"})
@@ -1486,25 +1565,10 @@ class Engine:
                 for j in jobs:
                     if j.label not in order:
                         order.append(j.label)
-                for label in order:
-                    mine = [j for j in jobs if j.label == label and j.status == "waiting"]
-                    if not mine:
-                        continue
-                    self._skip.clear()
-                    try:
-                        try:
-                            self._run_model(mine)
-                        except (Cancelled, ModelFailed):
-                            raise
-                        except Exception as e:  # never let one model's surprise kill the whole queue
-                            raise ModelFailed(f"unexpected error: {e!r}") from e
-                    except ModelFailed as e:
-                        for j in mine:
-                            if j.status in ("waiting", "loading", "running"):
-                                j.status = "skipped" if "by user" in str(e) else "failed"
-                                j.note, j.finished = str(e).splitlines()[0], time.time()
-                                self.emit("job_done", job=j)
-                        self.emit("model_failed", label=label, message=str(e))
+                groups = [(label, [j for j in jobs if j.label == label and j.status == "waiting"]) for label in order]
+                self._run_lanes([g for g in groups if g[1]], max(1, int(parallel or self.parallel_models())))
+            if self._cancel.is_set():
+                raise Cancelled()
         except Cancelled:
             for j in jobs:
                 if j.status in ("waiting", "loading", "running"):
@@ -1522,6 +1586,112 @@ class Engine:
                 self.emit("verdicts", changes=changes)
             self.emit("queue_done", jobs=jobs, cancelled=self._cancel.is_set())
 
+    def _run_lanes(self, groups, parallel):
+        """Run each model's jobs in its own lane, up to `parallel` at a time, in order: the next
+        model starts once a lane is free, nothing running stops it (runs_alone), and its memory fits
+        next to the ones running (memory_need_gb; an unknown need is left to the setting)."""
+        pending, running, told = list(groups), {}, set()
+        cond = threading.Condition()
+
+        def blocked(label):
+            """Why the next model can't start now (None: it can)."""
+            if not running:
+                return None
+            if len(running) >= parallel:
+                return ""
+            m = self.model(label)
+            if self.runs_alone(m) or any(self.runs_alone(self.model(l)) for l in running):
+                alone = label if self.runs_alone(m) else next(l for l in running if self.runs_alone(self.model(l)))
+                return (f"{alone}'s server runs a before_start step that may stop other servers, so it runs "
+                        "on its own")
+            need = self.memory_need_gb(m)
+            used = sum(n for n in running.values() if n)
+            room = self.memory_available_gb()
+            if need and used + need > room:
+                return (f"needs ~{need:.0f} GB; {', '.join(running)} take ~{used:.0f} of {room:.0f} GB, so it "
+                        "starts when there's room")
+            return None
+
+        def lane():
+            while True:
+                with cond:
+                    while True:
+                        if self._cancel.is_set() or not pending:
+                            return
+                        label = pending[0][0]
+                        why = blocked(label)
+                        if why is None:
+                            label, mine = pending.pop(0)
+                            running[label] = self.memory_need_gb(self.model(label))
+                            break
+                        if why and label not in told:
+                            told.add(label)
+                            self.emit("model_waiting", label=label, message=f"{label} waits: {why}")
+                        cond.wait(1.0)
+                try:
+                    self._run_lane(label, mine)
+                finally:
+                    with cond:
+                        running.pop(label, None)
+                        cond.notify_all()
+
+        if parallel == 1 or len(groups) == 1:
+            lane()
+            return
+        threads = [threading.Thread(target=lane, name=f"lane-{i + 1}") for i in range(min(parallel, len(groups)))]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join()
+
+    def _run_lane(self, label, mine):
+        """One model's jobs, in this thread's own lane. Other models served meanwhile are noted on
+        each answer (ran_alongside), since they share the machine's GPU and memory bandwidth."""
+        lane = Lane(label)
+        m = mine[0].model
+        with self._lanes_lock:
+            for other in self._lanes.values():
+                other.seen.add(label)
+            lane.seen = set(self._lanes)
+            if self.cfg["servers"][m["server"]].get("cmd"):
+                lane.port = self._lane_port(m)
+            self._lanes[label] = lane
+        self._here.lane = lane
+        try:
+            try:
+                self._run_model(mine)
+            except (Cancelled, ModelFailed):
+                raise
+            except Exception as e:  # never let one model's surprise kill the whole queue
+                raise ModelFailed(f"unexpected error: {e!r}") from e
+        except ModelFailed as e:
+            for j in mine:
+                if j.status in ("waiting", "loading", "running"):
+                    j.status = "skipped" if "by user" in str(e) else "failed"
+                    j.note, j.finished = str(e).splitlines()[0], time.time()
+                    self.emit("job_done", job=j)
+            self.emit("model_failed", label=label, message=str(e))
+        except Cancelled:
+            pass   # run() marks what's left
+        finally:
+            self._here.lane = None
+            with self._lanes_lock:
+                self._lanes.pop(label, None)
+
+    def _lane_port(self, m):
+        """The server's port, or a free one when another model being served has it (call with
+        _lanes_lock held)."""
+        port = self.cfg["servers"][m["server"]].get("port")
+        if port is None or not any(l.port == port for l in self._lanes.values()):
+            return port
+        taken = {l.port for l in self._lanes.values()}
+        while True:
+            with socket.socket() as s:
+                s.bind(("127.0.0.1", 0))
+                free = s.getsockname()[1]
+            if free not in taken:
+                return free
+
     @contextlib.contextmanager
     def serve(self, m, perf_args=None, port=None, log_name=None):
         """Start a model's server (or use the running one), wait until it serves the expected name,
@@ -1649,11 +1819,16 @@ class Engine:
         for j in jobs:
             j.status = "loading"
             self.emit("job_update", job=j)
-        with self.serve(m) as (base_url, info):
+        lane = self._lane()
+        with self.serve(m, port=lane.port) as (base_url, info):
+            with self._lanes_lock:
+                loaded_alongside = sorted(lane.seen)   # served while this one loaded: its load time is shared
+
             def run_info():
                 return {"load_s": info["load_s"], "peak_rss_mb": info["peak_mb"](),
                         "server_facts": info["facts"] or None, "gpu_peak_gb": info.get("gpu_peak_gb"),
-                        "kernel_share_max": info.get("kernel_share_max"), **serving_info}
+                        "kernel_share_max": info.get("kernel_share_max"),
+                        **({"loaded_alongside": loaded_alongside} if loaded_alongside else {}), **serving_info}
             failed_in_row = 0
             for j in jobs:
                 self._check()
@@ -1736,7 +1911,7 @@ class Engine:
         return tests
 
     # ---- sittings: a pack run over several sessions (stopped and resumed) keeps its total time
-    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "kernel_share_max", "machine")
+    SITTING_KEYS = ("load_s", "peak_rss_mb", "gpu_peak_gb", "kernel_share_max", "machine", "loaded_alongside")
 
     def _sitting(self, job, info):
         now = time.time()
@@ -1811,6 +1986,9 @@ class Engine:
         for k in ("peak_rss_mb", "gpu_peak_gb", "kernel_share_max"):
             if peak(k) is not None:
                 out[k] = peak(k)
+        along = sorted({l for s in sittings for l in s.get("loaded_alongside") or []})
+        if along:
+            out["loaded_alongside"] = along
         return out
 
     def _run_job(self, job, base_url, run_info, alive=lambda: True):
@@ -1862,6 +2040,9 @@ class Engine:
                 if (test["id"], rep) in done_keys:
                     continue
                 self._check()
+                lane = self._lane()
+                with self._lanes_lock:
+                    lane.seen = set(self._lanes) - {job.label}
                 self.emit("request_started", job=job, test=test["description"], test_id=test["id"],
                           has_image=bool(test.get("image")), repeat=rep, difficulty=test.get("difficulty", "unrated"))
                 waits = list(self.SERVER_RETRY_WAITS_S)
@@ -1870,7 +2051,7 @@ class Engine:
                     try:
                         body = self._body(job, test, rep)
                         res = client.stream_chat(base_url, body,
-                                                 lambda kind, text: self.emit("delta", stream=kind, text=text),
+                                                 lambda kind, text: self.emit("delta", stream=kind, text=text, label=job.label),
                                                  timeout=timeout, stream=self._stream, headers=headers)
                     except client.Cancelled:
                         self._check()
@@ -1917,6 +2098,8 @@ class Engine:
                        **({"checks": g["checks"]} if "checks" in g else {}),
                        "machine": self.machine().id,
                        **({"quantization": ep.get("quantization")} if ep else {}),
+                       # other models served while this answer ran: they shared the machine, so its speed did too
+                       **({"ran_alongside": sorted(lane.seen)} if lane.seen else {}),
                        "sent": "+".join(msg["role"] for msg in body["messages"]),  # proof: single turn
                        **({"seed": body["seed"]} if "seed" in body else {}), **meta}
                 # The same long answer twice for one question, with sampling on, means a cached response

tuieval/graders/code.py

@@ -53,6 +53,7 @@ def _err(e):
     return last[:200]
 try:
     exec(compile(CANDIDATE, "candidate.py", "exec"), ns)
+    ns = dict(ns)  # tests get a copy, so their imports can't rebind the candidate's globals
     exec(compile(SETUP, "setup.py", "exec"), ns)
     setup_error = None
 except BaseException as e:

tuieval/run_evals.py

@@ -15,6 +15,7 @@
     tuieval run --tier smoke --only my-model      # 3 tests per pack, 1 repeat: check paths and flags
     tuieval run --tier smoke --only openrouter:qwen/qwen3-32b   # any OpenRouter model, no models.toml edit
     tuieval run --dry-run                         # print the plan and server commands only
+    tuieval run --parallel 2                      # serve two models at a time (parallel_models in models.toml)
     tuieval add ~/models/New-Model-Q4_K_M.gguf    # register a model (--vision/--mmproj, --no-think, --tags)
     tuieval scan                                  # GGUFs in model_dirs that aren't registered yet
     tuieval list                                  # the models tuieval knows; hidden ones listed separately
@@ -53,6 +54,7 @@ class Printer:
     def __init__(self):
         self.lock = threading.Lock()
         self.mode = None
+        self.side_by_side = False   # several models at a time: one line per answer, no live streams
 
     def __call__(self, kind, **d):
         with self.lock:
@@ -73,6 +75,9 @@ class Printer:
     def on_server_stall(self, label, message):
         say(f"STALL: {message}", RED)
 
+    def on_model_waiting(self, label, message):
+        say(message, YELLOW)
+
     def on_server_retry(self, label, message):
         say(message, RED)
 
@@ -97,11 +102,15 @@ class Printer:
             + (f" ({earlier_s / 60:.0f} min over {sittings} earlier sitting(s))" if sittings else ""))
 
     def on_request_started(self, job, test, test_id, has_image, repeat, difficulty="unrated"):
+        if self.side_by_side:
+            return
         tag = {"easy": GREEN, "medium": "\033[33m", "hard": RED}.get(difficulty, DIM) + f"[{difficulty}]" + RESET
         print(f"\n{BOLD}{CYAN}━━ {job.key} [{job.done + 1}/{job.total}]{RESET} {tag} {BOLD}{CYAN}{test}{RESET}", flush=True)
         self.mode = None
 
-    def on_delta(self, stream, text):
+    def on_delta(self, stream, text, label=None):
+        if self.side_by_side:
+            return
         if self.mode != stream:
             sys.stdout.write(f"{DIM}[thinking] " if stream == "reasoning" else f"{RESET}\n{BOLD}[answer]{RESET} ")
             self.mode = stream
@@ -114,8 +123,10 @@ class Printer:
         calls = f" tool={r['tool_calls'][0]['name']}" if r.get("tool_calls") else ""
         speed = f"{r['gen_tps']:.0f} tok/s, " if r.get("gen_tps") else ""
         ttft = f"TTFT {r['ttft_s']:.1f}s, " if r.get("ttft_s") is not None else ""
-        print(f"{RESET}\n{mark} {r['description']}{RESET}{calls} {DIM}({r.get('completion_tokens')} tokens, "
-              f"{ttft}{speed}{r['total_s']:.1f}s) {r['reason'][:120]}{RESET}", flush=True)
+        where = f"{CYAN}{job.key} [{job.done}/{job.total}]{RESET} " if self.side_by_side else ""
+        print(f"{RESET}{'' if self.side_by_side else chr(10)}{where}{mark} {r['description']}{RESET}{calls} "
+              f"{DIM}({r.get('completion_tokens')} tokens, {ttft}{speed}{r['total_s']:.1f}s) {r['reason'][:120]}{RESET}",
+              flush=True)
 
     def on_job_done(self, job):
         color = GREEN if job.status == "done" else RED if job.status == "failed" else ""
@@ -683,6 +694,8 @@ def main(argv=None):
     p.add_argument("--tier", choices=engine.TIERS, default="screen",
                    help="smoke: 3 tests; screen (default): a spread sample; certify: everything, with repeats")
     p.add_argument("--smoke", action="store_const", const="smoke", dest="tier", help="same as --tier smoke")
+    p.add_argument("--parallel", type=int, metavar="N",
+                   help="models served at a time (default: parallel_models for this machine in models.toml, else 1)")
     p.add_argument("--dry-run", action="store_true", help="print the plan and commands without running")
     p.add_argument("--brief", action="store_true", help="end with one line per model instead of the scorecards")
     p.add_argument("--models", default=None, help="default: the workspace's models.toml")
@@ -691,8 +704,12 @@ def main(argv=None):
     p.add_argument("--log-dir", help="server and request logs (default: logs/)")
     args = p.parse_args(argv)
 
-    e = engine.Engine(Printer(), models_path=args.models, packs_dir=args.packs_dir,
+    printer = Printer()
+    e = engine.Engine(printer, models_path=args.models, packs_dir=args.packs_dir,
                       results_dir=args.results_dir, log_dir=args.log_dir)
+    if args.parallel is not None and args.parallel < 1:
+        sys.exit("--parallel must be 1 or more")
+    parallel = args.parallel or e.parallel_models()
     models = e.cfg["models"]
     labels = [m["label"] for m in models if not m.get("remote")]   # hosted APIs cost money: only by name
     if args.only:
@@ -757,7 +774,12 @@ def main(argv=None):
 
     signal.signal(signal.SIGINT, stop)
     signal.signal(signal.SIGTERM, stop)
-    worker = threading.Thread(target=e.run, args=(jobs,))
+    side_by_side = min(parallel, len({j.label for j in jobs if j.status == "waiting"}))
+    if side_by_side > 1:
+        printer.side_by_side = True
+        say(f"up to {side_by_side} models at a time: one line per answer; each answer notes the models "
+            "it ran alongside, since they share this machine's speed")
+    worker = threading.Thread(target=e.run, args=(jobs, parallel))
     worker.start()
     while worker.is_alive():
         worker.join(0.5)

tuieval/tui.py

@@ -290,6 +290,9 @@ HELP = {
 2. Pick a [b]tier[/b]: Smoke first to check the setup, then Screen, then Certify for finalists.
 3. Press [b]s[/b]. The line above the buttons says how many answers that is and roughly how long.
 
+[b]Models at a time[/b]: how many of the ticked models run side by side (blank: this machine's
+parallel_models in models.toml, default 1). Answers note the models they ran alongside.
+
 [b]Keys[/b]
   s  start (or queue it behind a run that's going)     r  results
   a  add a model (a GGUF, a model id, openrouter:<id>)  m  scan model folders for new GGUFs
@@ -308,7 +311,8 @@ with the model's reasoning and answer as they stream, then recent results with t
 [b]Keys[/b]
   k  skip the current model          c  cancel everything (finished answers are kept; it resumes next time)
   f  pause or resume auto-scroll     r  results     n  set up a new run (this one keeps going)
-  w  runs and queue                  esc  back to Setup (the run keeps going)""",
+  w  runs and queue                  esc  back to Setup (the run keeps going)
+  v  with several models at a time: stream the next one (k skips the one streaming)""",
     "ResultsScreen": """[b]Results[/b]
 
 Start with [b]Production readiness[/b]: one verdict per model and use case, and what's missing when
@@ -435,6 +439,7 @@ class AnswerScreen(ModalScreen):
                  f"{rec['gen_tps']:.0f} tok/s" if rec.get("gen_tps") else "",
                  f"{rec['total_s']:.0f}s total" if rec.get("total_s") is not None else "",
                  f"on {rec['machine']}" if rec.get("machine") else "",
+                 f"[yellow]ran alongside {', '.join(rec['ran_alongside'])}[/yellow]" if rec.get("ran_alongside") else "",
                  f"seed {rec['seed']}" if "seed" in rec else ""]
         self.query_one("#answer-head", Static).update(
             f"{mark}  {diff_badge(rec.get('difficulty', 'unrated'))}  [b]{rich_escape(rec.get('description', rec['test']))}[/b]"
@@ -544,6 +549,8 @@ class SetupScreen(Screen):
         with Horizontal(id="options"):
             yield Label("Repeats")
             yield Input("", id="repeat", type="integer", max_length=2, placeholder="1")
+            yield Label("Models at a time")
+            yield Input("", id="parallel", type="integer", max_length=1, placeholder="1")
             yield Checkbox("Rerun existing results", id="force")
         yield Static(id="estimate")
         with Horizontal(id="buttons"):
@@ -761,6 +768,13 @@ class SetupScreen(Screen):
                 list(self.query_one("#suites", SelectionList).selected), repeat,
                 self.tier(), self.query_one("#force", Checkbox).value)
 
+    def parallel(self):
+        """Models at a time for the next run: the field, or blank for this machine's parallel_models."""
+        try:
+            return max(1, int(self.query_one("#parallel", Input).value)) or None
+        except ValueError:
+            return None
+
     def current_picks(self, suites=None):
         """{pack: picked test ids} for packs that still have those tests (and, given suites, are ticked)."""
         e = self.app.engine
@@ -803,6 +817,7 @@ class SetupScreen(Screen):
         e = self.app.engine
         defaults = sorted({e.default_repeat(e.packs[p], tier) for p in suites}) or [1]
         self.query_one("#repeat", Input).placeholder = str(defaults[0]) if len(defaults) == 1 else "pack"
+        self.query_one("#parallel", Input).placeholder = str(e.parallel_models())
         if not e.packs:
             est.update("[yellow]No packs yet.[/yellow] [dim]Quit, run tuieval new-pack <name>, "
                        "and open tuieval again.[/dim]")
@@ -852,6 +867,11 @@ class SetupScreen(Screen):
         resumed = [j for j in run if "done earlier" in j.note]
         if resumed:
             text += f"   [yellow]{len(resumed)} pack(s) continue from earlier results[/yellow]"
+        at_once = min(self.parallel() or e.parallel_models(), len({j.label for j in run}))
+        if at_once > 1:
+            text += (f"\n[cyan]Up to {at_once} models at a time[/cyan][dim] (the time above assumes one at a "
+                     "time). Answers note the models they ran alongside, since those share the machine's "
+                     "speed.[/dim]")
         est.update(text + self.queue_hint())
 
     def queue_hint(self):
@@ -879,6 +899,8 @@ class SetupScreen(Screen):
             if name in e.packs:
                 packs.select(name)
         self.query_one("#repeat", Input).value = str(sel["repeat"]) if sel.get("repeat") else ""
+        if "parallel" in sel:
+            self.query_one("#parallel", Input).value = str(sel["parallel"]) if sel["parallel"] else ""
         tier = sel.get("tier") or ("smoke" if sel.get("smoke") else None)
         if tier in engine.TIERS:
             self.query_one(f"#tier-{tier}", RadioButton).value = True
@@ -917,7 +939,8 @@ class SetupScreen(Screen):
             self.notify("Select at least one eval pack and one model.", severity="warning")
             return
         picks = self.current_picks(suites)
-        sel = {"models": labels, "packs": suites, "repeat": repeat, "tier": tier, "force": force, "tests": picks}
+        sel = {"models": labels, "packs": suites, "repeat": repeat, "tier": tier, "force": force, "tests": picks,
+               "parallel": self.parallel()}
         jobs = self.app.engine.plan(labels, suites, repeat, tier, force, picks)
         rerun = False
         if not any(j.status == "waiting" for j in jobs):
@@ -1453,7 +1476,7 @@ class QueuedRun:
         if new:
             return None, (f"it would now start over {', '.join(new)}, which wasn't confirmed when it was queued. "
                           "Start it again from Setup to see why")
-        return RunScreen(jobs), None
+        return RunScreen(jobs, sel.get("parallel")), None
 
 
 class QueuedTune:
@@ -1783,6 +1806,7 @@ class RunScreen(Screen):
         Binding("k", "skip_model", "Skip model", priority=True),
         Binding("c", "cancel_all", "Cancel all", priority=True),
         Binding("f", "toggle_follow", "Pause scroll", priority=True),
+        Binding("v", "watch_next", "Next model", priority=True),
         Binding("n", "new_run", "New run", priority=True),
         Binding("r", "results", "Results", priority=True),
         Binding("w", "runs", "Runs", priority=True),
@@ -1793,9 +1817,11 @@ class RunScreen(Screen):
     LIVE_ONLY = {"skip_model", "cancel_all", "toggle_follow"}
     DONE_ONLY = {"new_run"}
 
-    def __init__(self, jobs):
+    def __init__(self, jobs, parallel=None):
         super().__init__()
         self.jobs = jobs
+        self.parallel = parallel      # models at a time (None: this machine's parallel_models)
+        self.watch = None             # the model whose answers stream (several run at a time)
         self.running = True
         self.follow = True
         self.buf = {"reasoning": [], "answer": []}
@@ -1839,12 +1865,24 @@ class RunScreen(Screen):
         self.notify("\n".join(lines), title=title, severity="error", timeout=30)
 
     def check_action(self, action, parameters):
+        if action == "watch_next":
+            return self.running and self.side_by_side
         if action in self.LIVE_ONLY:
             return self.running
         if action in self.DONE_ONLY:
             return not self.running
         return True
 
+    @property
+    def side_by_side(self):
+        """Several models run at a time in this run."""
+        return min(self.parallel or self.app.engine.parallel_models(),
+                   len({j.label for j in self.jobs if j.status != "skipped"})) > 1
+
+    def active(self):
+        """Models being loaded or run now, in queue order."""
+        return list(dict.fromkeys(j.label for j in self.jobs if j.status in ("loading", "running")))
+
     def counts(self):
         return (sum(j.done for j in self.jobs), sum(j.passed for j in self.jobs), sum(j.failed for j in self.jobs))
 
@@ -1908,14 +1946,18 @@ class RunScreen(Screen):
         self.query_one("#bar", ProgressBar).update(total=max(self.total, 1), progress=0)
         self.set_interval(0.1, self.flush_streams)
         self.set_interval(1.0, self.tick)
-        self.app.start_run(self.jobs, self.on_engine_event)
+        self.app.start_run(self.jobs, self.on_engine_event, self.parallel)
         self.tick()
 
     # -- events from the engine thread
     def on_engine_event(self, kind, **d):
         if self.app.closing:
             return  # the app is shutting down; the UI thread may be waiting on the engine
+        if kind in ("request_started", "model_loading") and self.watch not in self.active():
+            self.watch = d["job"].label if "job" in d else d["label"]   # follow the next model streaming
         if kind == "delta":  # very frequent: buffer, flushed every 100 ms on the UI thread
+            if d.get("label", self.watch) != self.watch:
+                return       # another model running alongside: v switches to it
             with self.buf_lock:
                 self.buf[d["stream"]].append(d["text"])
             return
@@ -1936,8 +1978,15 @@ class RunScreen(Screen):
     def event(self, text):
         self.query_one("#events", Log).write_line(time.strftime("%H:%M:%S ") + text)
 
+    def watched(self, kind, d):
+        """Whether an event belongs to the streaming model (always, with one model at a time)."""
+        label = d["job"].label if "job" in d else d.get("label")
+        return label is None or label == self.watch or not self.side_by_side
+
     def handle(self, kind, d):
-        if kind == "request_started":
+        if kind == "request_started" and not self.watched(kind, d):
+            pass         # another model running alongside: its rows still show in Recent results
+        elif kind == "request_started":
             self.flush_streams()
             self.query_one("#reasoning", StreamView).clear()
             self.query_one("#answer", StreamView).clear()
@@ -1948,7 +1997,8 @@ class RunScreen(Screen):
                             f"{'  [magenta]🖼 image[/magenta]' if d['has_image'] else ''}\n"
                             f"{diff_badge(d.get('difficulty', 'unrated'))}  {d['test']}")
         elif kind == "request_done":
-            self.flush_streams()
+            if self.watched(kind, d):
+                self.flush_streams()
             job, r = d["job"], d["record"]
             mark = "[green]✓[/green]" if r["pass"] else "[red]✗[/red]"
             recent = self.query_one("#recent", DataTable)
@@ -1974,7 +2024,8 @@ class RunScreen(Screen):
                 recent.move_cursor(row=recent.row_count - 1)
             if job:
                 self.update_job(job)
-            self.req_started = None
+            if self.watched(kind, d):
+                self.req_started = None
             self.update_overall()
         elif kind in ("job_started", "job_update", "job_done"):
             job = d["job"]
@@ -2008,15 +2059,18 @@ class RunScreen(Screen):
             self.current = "The other window finished; starting…"
             self.event("the machine is free; starting")
         elif kind == "model_loading":
-            self.current = f"Loading [b]{d['label']}[/b]…"
             self.event(f"{d['label']}: starting server")
             if d.get("log"):
                 self.server_logs[d["label"]] = d["log"]
-            self.query_one("#serverlog", Log).write_line(f"$ {' '.join(d['command'])}")
-            self.query_one("#reasoning", StreamView).clear()
-            self.query_one("#answer", StreamView).clear()
+            self.query_one("#serverlog", Log).write_line(self.log_prefix(d) + f"$ {' '.join(d['command'])}")
+            if self.watched(kind, d):
+                self.current = f"Loading [b]{d['label']}[/b]…"
+                self.query_one("#reasoning", StreamView).clear()
+                self.query_one("#answer", StreamView).clear()
+        elif kind == "model_waiting":
+            self.event(f"[yellow]{d['message']}[/yellow]")
         elif kind == "server_log":
-            self.query_one("#serverlog", Log).write_line(d["line"])
+            self.query_one("#serverlog", Log).write_line(self.log_prefix(d) + d["line"])
         elif kind == "server_stall":
             self.event(f"[bold red]STALL[/bold red] {d['message']}")
             self.notify(d["message"], severity="error", timeout=30)
@@ -2029,7 +2083,8 @@ class RunScreen(Screen):
         elif kind == "model_ready":
             loaded = f" in {d['load_s']:.0f}s" if d.get("load_s") else ""
             self.event(f"{d['label']}: ready as {d['ids']}{loaded}")
-            self.current = f"[b]{d['label']}[/b] ready{loaded}, starting evals…"
+            if self.watched(kind, d):
+                self.current = f"[b]{d['label']}[/b] ready{loaded}, starting evals…"
         elif kind == "resource":
             self.mem[d["label"]] = d["rss_mb"]
         elif kind == "server_retry":
@@ -2082,6 +2137,9 @@ class RunScreen(Screen):
             self.app.session_finished(self)
         self.refresh_current()
 
+    def log_prefix(self, d):
+        return f"[{d['label']}] " if self.side_by_side and d.get("label") else ""
+
     def update_job(self, j):
         q = self.query_one("#queue", DataTable)
         style = STATUS_STYLE.get(j.status, "")
@@ -2115,7 +2173,9 @@ class RunScreen(Screen):
             speed += f" · {sorted(tps)[len(tps) // 2]:.0f} tok/s"
         if ttfts:
             speed += f" · TTFT {sorted(ttfts)[len(ttfts) // 2]:.1f}s"
-        mem = max(self.mem.values()) if self.mem else 0
+        active = self.active()
+        mem = (sum(self.mem.get(l, 0) for l in active) if self.side_by_side and active   # models side by side add up
+               else max(self.mem.values()) if self.mem else 0)
         if mem:
             speed += f" · mem {mem / 1024:.1f} GB" if mem >= 1024 else f" · mem {mem:.0f} MB"
         self.query_one("#overall", Static).update(
@@ -2132,6 +2192,12 @@ class RunScreen(Screen):
             if self.req_first and now - self.req_first > 1:
                 rate = f" · ~{self.req_tokens / (now - self.req_first):.0f} tok/s"
             extra = f"   [dim]{fmt_secs(now - self.req_started)} · {ttft}~{self.req_tokens} tokens{rate}[/dim]"
+        others = [l for l in self.active() if l != self.watch] if self.running and self.side_by_side else []
+        if others:
+            def where(label):
+                j = next((j for j in self.jobs if j.label == label and j.status in ("loading", "running")), None)
+                return f"{label} ({j.pack.label} {j.done}/{j.total})" if j and j.status == "running" else f"{label} (loading)"
+            extra += f"\n[dim]Also running: {' · '.join(map(where, others))} · v streams the next one[/dim]"
         self.query_one("#current", Static).update(self.current + extra)
 
     def tick(self):
@@ -2144,8 +2210,28 @@ class RunScreen(Screen):
     # -- actions
     def action_skip_model(self):
         if self.running:
-            self.app.engine.skip_model()
-            self.notify("Skipping the current model…")
+            if self.side_by_side and self.watch:
+                self.app.engine.skip_model(self.watch)
+                self.notify(f"Skipping {self.watch}; the others keep going…")
+            else:
+                self.app.engine.skip_model()
+                self.notify("Skipping the current model…")
+
+    def action_watch_next(self):
+        """Stream the next model running alongside (from its next answer)."""
+        active = self.active()
+        if len(active) < 2:
+            self.notify("Only one model is running right now.")
+            return
+        self.watch = active[(active.index(self.watch) + 1) % len(active)] if self.watch in active else active[0]
+        with self.buf_lock:
+            for v in self.buf.values():
+                v.clear()
+        self.query_one("#reasoning", StreamView).clear()
+        self.query_one("#answer", StreamView).clear()
+        self.req_started = None
+        self.current = f"Streaming [b]{self.watch}[/b] from its next answer…"
+        self.refresh_current()
 
     def action_cancel_all(self):
         if not self.running:
@@ -2548,6 +2634,11 @@ class ResultsScreen(Screen):
                      f"failed; both passed {h['both']}, both failed {h['neither']}. Fewer tokens: {a} on "
                      f"{h['fewer_a']}, {b} on {h['fewer_b']}" + (f" (median {a}/{b} ratio {h['ratio']:.2f}×)"
                                                                if h["ratio"] else "") + ".")
+        shared_speed = sorted({c_m for t in shared for c_m, c in t["cells"].items() if c_m in models
+                               and any(r.get("alongside") for r in c["rows"])})
+        if shared_speed:
+            line += (f"\n[yellow]Some answers of {', '.join(shared_speed)} ran alongside other models, which "
+                     "shared the machine's speed: compare their times with care.[/yellow]")
         title.update(line + "  [dim]Fewest tokens / Fastest = questions won; ties count for both. Total tokens / "
                      "Total time add up each question's median. Avg tok/s = all tokens over all the time they took "
                      "(cached prompts left out).[/dim]")
@@ -2676,6 +2767,7 @@ class EvalsApp(App):
     #options { height: 3; padding: 0 1; align-vertical: middle; }
     #options Label { padding: 1 1 0 0; }
     #options Input { width: 12; }
+    #options #parallel { width: 8; }
     #estimate { padding: 0 2; height: auto; min-height: 2; }
     #buttons { height: 3; padding: 0 1; }
     #buttons Button { margin-right: 2; }
@@ -2965,10 +3057,10 @@ class EvalsApp(App):
         self.worker = threading.Thread(target=work, name="tune")
         self.worker.start()
 
-    def start_run(self, jobs, on_event):
+    def start_run(self, jobs, on_event, parallel=None):
         self.engine.on_event = on_event
         # A plain (non-daemon) thread, so quitting can wait for servers to be stopped.
-        self.worker = threading.Thread(target=self.engine.run, args=(jobs,), name="engine")
+        self.worker = threading.Thread(target=self.engine.run, args=(jobs, parallel), name="engine")
         self.worker.start()
 
     def on_unmount(self):

{tuieval-0.2.0.dev23.dist-info → tuieval-0.2.0.dev28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev23
+Version: 0.2.0.dev28
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev23.dist-info → tuieval-0.2.0.dev28.dist-info}/RECORD

@@ -1,22 +1,22 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=hrYSfDsgWa6h-shaGODbtlY3KfUtC003qRSAak6fmuw,535
+tuieval/_version.py,sha256=mCD5WpGBJi5o6Jnxl_I1TiLi_4xG9de502jCBwE897Y,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=7joEA07W2we-kcg4VXxfCSXazj7dWBCKlQlSXpXQfTc,4390
 tuieval/client.py,sha256=Vui7ucVv_GiKObGoVS_EWSZFUKPou9PWmutqMTrahXk,9313
-tuieval/compare.py,sha256=b-rVcMwq0Tos3oNKcpUP1HXULKhAQNAS4rlXYSY7t6k,25460
+tuieval/compare.py,sha256=SHN_o_nlppFcHMBqBZRITIkeQDPaR_JxhUZItQLMBwo,26270
 tuieval/doctor.py,sha256=y1C_ggjd8oikXTrovgTBf26wjThP8EoVkfTVeO_n5dQ,6070
-tuieval/engine.py,sha256=aRZn-PgB8e4DCLJNPsMhIsF6Kk-CHB_CIsP2ECRizy0,103183
+tuieval/engine.py,sha256=k9DtWDJLfTXSPMaIR4qw-0SadTD0ts5hb3Uv38oTY4g,111278
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/machines.py,sha256=ImEihNj8QlBRdCHLED4QIdjJUvjVtZgo6DiVU5h6Zzs,13415
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=nGkbMj0qE4G4RSgIK1lradOFD_8weITk0w54TRNxqI0,42426
+tuieval/run_evals.py,sha256=ZGx4XcZ15qD3vIg15xCfdX4SxTDHXBp_GYoiGexchdw,43668
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=aG-KMsD_8ma27fsHvw01S7V_g10pzJ-nrWi5pq28DA8,151175
+tuieval/tui.py,sha256=pMH15LkHEeIdr8GlB55Znxdy4OuTFDtNTkGrp7ItlvI,156704
 tuieval/tune.py,sha256=jXwYuSjfdqbLCE3WonP2xaUHwk_FnhI67twB1BaHggI,38992
 tuieval/verdict.py,sha256=Muo1a05mCv781MlEJsXwDsGaKbRi0JS4Lv8HdvK_Aco,19054
 tuieval/watch_proxy.py,sha256=z916NRhEur04a9RnLm6ZpIEY1SnnUGPiHYbmlD9ktQ0,11035
@@ -24,7 +24,7 @@ tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
 tuieval/yamlout.py,sha256=r1IokjLqW-Ztpg1fSn33ZW3VK00v1VvxymmU5Qy7GEw,606
 tuieval/graders/__init__.py,sha256=_YCFY5euaInIQdX4RoQGS3kMuiK6q5J7HF2e4i0ou3Q,5940
 tuieval/graders/answer.py,sha256=pTETs2LzOWPhtuuowMVXpLFwdnNjMw-7sELqrFV6ocQ,2125
-tuieval/graders/code.py,sha256=mNoPw-C_lNaXTTU57TJjhcVPTvdR400PxoE9Ow-tgHg,3703
+tuieval/graders/code.py,sha256=Q97J0pRhg4xPlhLJoF1theV3vAECJB8jkM6JQMZnIes,3796
 tuieval/graders/rag.py,sha256=j_gEjmaXmY_hgKu7-7pVQpQ8Cwb49gadnwFurih-1Hw,1447
 tuieval/graders/reply.py,sha256=BdkL7EWSSMoyjwCOFa97TTQMR1QyrpMH-ntcgjzDPEg,2199
 tuieval/graders/tool_call.py,sha256=_QXURtOk4ZKBnxXVMzFL1c9jWLWksRlLZ4VqOCL0t90,3578
@@ -45,8 +45,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev23.dist-info/METADATA,sha256=fE4RF7BtlfvEDdZkR8sgZCUM7jpl9S0TXgFZg2076A4,13227
-tuieval-0.2.0.dev23.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev23.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev23.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev23.dist-info/RECORD,,
+tuieval-0.2.0.dev28.dist-info/METADATA,sha256=rVlRwOpwJXIudx4BD9v-tcTnH4RMb4SqOhtnLut5Xos,13227
+tuieval-0.2.0.dev28.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev28.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev28.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev28.dist-info/RECORD,,