tuieval 0.2.0.dev50__py3-none-any.whl → 0.2.0.dev56__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev50'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev50')
+__version__ = version = '0.2.0.dev56'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev56')
 
 __commit_id__ = commit_id = None

tuieval/engine.py

@@ -987,16 +987,16 @@ class Engine:
                     del per_pack[name]
         return {l: p for l, p in out.items() if p}
 
-    def estimate_seconds(self, jobs):
-        """(seconds, notes): expected run time for the waiting jobs, estimated per pack.
+    def seconds_per_answer(self, jobs, waiting_only=True):
+        """({job key: expected seconds per answer}, borrowed, guessed, thin) for the waiting jobs.
         A pack the model has answered uses its own median time there. Otherwise the median of
         other models' times on that pack, scaled by how much slower or faster this model is than
-        them on packs both have answered. notes lists what made the estimate rough ([] = solid)."""
+        them on packs both have answered. The sets say which estimates are rough (see estimate_seconds)."""
         times = self.answer_times()
         med = {l: {p: statistics.median(ts) for p, ts in pp.items()} for l, pp in times.items()}
-        total, borrowed, guessed, thin = 0.0, set(), set(), set()
+        spr_of, borrowed, guessed, thin = {}, set(), set(), set()
         for j in jobs:
-            if j.status != "waiting":
+            if j.status in ("done", "skipped", "failed") or (waiting_only and j.status != "waiting"):
                 continue
             own = med.get(j.label, {})
             ref = {p: statistics.median(v[p] for l, v in med.items() if p in v and l != j.label)
@@ -1013,8 +1013,18 @@ class Engine:
             else:
                 spr = statistics.median(own.values()) if own else DEFAULT_SECONDS_PER_REQUEST
                 guessed.add((j.key, bool(own)))
-            done = len(self._done_keys(j)) if j.merge else 0
-            total += max(0, j.total - done) * spr
+            spr_of[j.key] = spr
+        return spr_of, borrowed, guessed, thin
+
+    def estimate_seconds(self, jobs):
+        """(seconds, notes): expected run time for the waiting jobs, estimated per pack
+        (seconds_per_answer). notes lists what made the estimate rough ([] = solid)."""
+        spr_of, borrowed, guessed, thin = self.seconds_per_answer(jobs)
+        total = 0.0
+        for j in jobs:
+            if j.key in spr_of:
+                done = len(self._done_keys(j)) if j.merge else 0
+                total += max(0, j.total - done) * spr_of[j.key]
         notes = []
         if borrowed:
             factors = sorted({f for _, f, _ in borrowed if f is not None})

tuieval/machines.py

@@ -65,10 +65,10 @@ def detect():
         # macOS default when unset: about 2/3 of RAM up to 36 GB, 3/4 above.
         limit = wired / 1024 if wired > 0 else ram * (0.75 if ram > 36 else 2 / 3)
         gpu_cores = None
-        try:
-            out = subprocess.run(["system_profiler", "SPDisplaysDataType"], capture_output=True, text=True,
-                                 timeout=15).stdout
-            m = re.search(r"Total Number of Cores:\s*(\d+)", out)
+        try:   # the GPU driver's own count: instant, where system_profiler can take seconds (CI's virtual Macs)
+            out = subprocess.run(["ioreg", "-rc", "AGXAccelerator", "-d1"], capture_output=True, text=True,
+                                 timeout=5).stdout
+            m = re.search(r'"gpu-core-count"\s*=\s*(\d+)', out)
             gpu_cores = int(m.group(1)) if m else None
         except (OSError, subprocess.TimeoutExpired):
             pass

tuieval/tui.py

@@ -1855,6 +1855,8 @@ class RunScreen(Screen):
         self.current = ""
         self.req_first = None
         self.stats = []
+        self.spr, self.rough = None, set()   # job key -> expected seconds per answer (ETA), rough ones
+        self.live = {}                        # job key -> seconds per answer in this session
         self.mem = {}
         self.outcome = ""  # "Finished" or "Cancelled" once the queue is done
         self.ended = None
@@ -2037,6 +2039,8 @@ class RunScreen(Screen):
                            + (f"  [red]copy of repeat {r['identical_to_repeat'] + 1}[/red]"
                               if r.get("identical_to_repeat") is not None else ""))
             self.stats.append(r)
+            if r.get("total_s"):
+                self.live.setdefault(job.key, []).append(r["total_s"])
             self.recent_entries[row_key] = {"model": job.label, "pack": job.pack.name, "record": r,
                                             "pack_fp": job.pack.fingerprint}
             if recent.row_count > 500:
@@ -2187,6 +2191,34 @@ class RunScreen(Screen):
                            ("note", short_note(j.note))):
             q.update_cell(j.key, col, value, update_width=True)
 
+    LIVE_ANSWERS = 5   # a pack's own times this session take over from its history after this many
+
+    def eta_seconds(self):
+        """(seconds, rough): time left, pack by pack: answers left × seconds per answer. A pack's own
+        average this session once it has LIVE_ANSWERS of them, before that its history (Engine.
+        seconds_per_answer: this model's past times there, or other models' scaled). Divided by the
+        models running side by side. rough: some packs have no history of their own yet."""
+        new = [j for j in self.jobs if self.spr is None or (j.key not in self.spr and j.status not in
+                                                           ("done", "skipped", "failed"))]
+        if new:   # at the start, and for evals added to the run
+            spr, borrowed, guessed, _ = self.app.engine.seconds_per_answer(new, waiting_only=False)
+            self.spr = {**(self.spr or {}), **spr}
+            self.rough |= {k for k, *_ in borrowed} | {k for k, _ in guessed}
+        left, rough, models = 0.0, False, set()
+        for j in self.jobs:
+            if j.status in ("done", "skipped", "failed") or j.done >= j.total:
+                continue
+            live = self.live.get(j.key, [])
+            if len(live) >= self.LIVE_ANSWERS:
+                spr = sum(live) / len(live)
+            else:
+                spr = self.spr.get(j.key, engine.DEFAULT_SECONDS_PER_REQUEST)
+                rough |= j.key in self.rough
+            left += (j.total - j.done) * spr
+            models.add(j.label)
+        lanes = max(1, min(self.parallel or self.app.engine.parallel_models(), len(models)))
+        return left / lanes, rough
+
     def update_overall(self):
         done = sum(j.done for j in self.jobs)
         passed = sum(j.passed for j in self.jobs)
@@ -2194,9 +2226,9 @@ class RunScreen(Screen):
         self.query_one("#bar", ProgressBar).update(progress=done)
         elapsed = time.time() - self.started
         eta = ""
-        done_now = sum(j.done - j.resumed for j in self.jobs if j.started)   # answers from earlier sittings
-        if self.running and done_now > 0 and self.total > done:              # took no time in this one
-            eta = f" · ETA {fmt_secs(elapsed / done_now * (self.total - done))}"
+        if self.running and self.total > done:
+            left, rough = self.eta_seconds()
+            eta = f" · ETA {'~' if rough else ''}{fmt_secs(left)}"
         pct = f"{100 * passed / (passed + failed):.0f}%" if passed + failed else "-"
         recent = self.stats[-50:]
         tps = [r["gen_tps"] for r in recent if r.get("gen_tps")]

tuieval/watch_proxy.py

@@ -18,6 +18,7 @@ import http.client
 import itertools
 import json
 import os
+import socketserver
 import sys
 import threading
 import time
@@ -129,6 +130,12 @@ class Proxy:
         class Server(ThreadingHTTPServer):
             daemon_threads = True
 
+            def server_bind(self):
+                # HTTPServer looks up its own hostname here (socket.getfqdn), which can take half a
+                # minute where DNS is slow; the name is only cosmetic, so use the address.
+                socketserver.TCPServer.server_bind(self)
+                self.server_name, self.server_port = self.server_address[:2]
+
             def handle_error(self, request, client_address):
                 err = sys.exc_info()[1]
                 if isinstance(err, (BrokenPipeError, ConnectionResetError)):

{tuieval-0.2.0.dev50.dist-info → tuieval-0.2.0.dev56.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev50
+Version: 0.2.0.dev56
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev50.dist-info → tuieval-0.2.0.dev56.dist-info}/RECORD

@@ -1,14 +1,14 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=cUlv5rRBNcBZ7a8L_s4uRBMTmFx7dza4nVKsHqXS0Pg,535
+tuieval/_version.py,sha256=w3V6noQtCnYcwkOc0Hi3EmCrxSS1deuXpcG9aisclJI,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=gzGBzhqvR0WKJQQNHB8CudpgViIeIyawrNWa54whmoI,4502
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
 tuieval/compare.py,sha256=SHN_o_nlppFcHMBqBZRITIkeQDPaR_JxhUZItQLMBwo,26270
 tuieval/doctor.py,sha256=y1C_ggjd8oikXTrovgTBf26wjThP8EoVkfTVeO_n5dQ,6070
-tuieval/engine.py,sha256=w3KsgxzFAMcRyVzFGQXhTgN3d9qYRtAZZxNLXZOgnrQ,114222
+tuieval/engine.py,sha256=0PuHFus1k02qAY8nlsI6z5iQo7IT82bMwmHDZT6fGtU,114784
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
-tuieval/machines.py,sha256=ImEihNj8QlBRdCHLED4QIdjJUvjVtZgo6DiVU5h6Zzs,13415
+tuieval/machines.py,sha256=tdRrn3WNOmCHwfcxcWz1NuAuOJdI7WnJFR1y6xgF89c,13512
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
@@ -17,10 +17,10 @@ tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=5bQEqU533XYiLze9dYcl72c5GSIpohMj1eqJrePRwnc,46450
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=21Re8fM5rf3zbHIKvK9laMV-mL2HjcGfMReoRg0DRLw,164268
+tuieval/tui.py,sha256=xAQnR270m7x-rfTdK7LBPMM9MEQC37zndD1TOTq8EUU,166103
 tuieval/tune.py,sha256=jXwYuSjfdqbLCE3WonP2xaUHwk_FnhI67twB1BaHggI,38992
 tuieval/verdict.py,sha256=Muo1a05mCv781MlEJsXwDsGaKbRi0JS4Lv8HdvK_Aco,19054
-tuieval/watch_proxy.py,sha256=z916NRhEur04a9RnLm6ZpIEY1SnnUGPiHYbmlD9ktQ0,11035
+tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
 tuieval/yamlout.py,sha256=r1IokjLqW-Ztpg1fSn33ZW3VK00v1VvxymmU5Qy7GEw,606
 tuieval/graders/__init__.py,sha256=_YCFY5euaInIQdX4RoQGS3kMuiK6q5J7HF2e4i0ou3Q,5940
@@ -46,8 +46,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev50.dist-info/METADATA,sha256=VW1tuB9QtgVUt9irfVVzrDhkF907wql1_WpM7PasfPU,13881
-tuieval-0.2.0.dev50.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev50.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev50.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev50.dist-info/RECORD,,
+tuieval-0.2.0.dev56.dist-info/METADATA,sha256=nUWInJiqgtyGg3hiyaYIUuGD-BYOzoyqi3kRoSloRPk,13881
+tuieval-0.2.0.dev56.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev56.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev56.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev56.dist-info/RECORD,,