tuieval 0.2.0.dev55__tar.gz → 0.2.0.dev56__tar.gz

{tuieval-0.2.0.dev55 → tuieval-0.2.0.dev56}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- The run's ETA adds up each pack's answers left × that model's time per answer there (its history, then its own times once it has 5 answers), instead of extrapolating from the first few answers.
 - Faster start on Macs: the GPU core count comes from `ioreg` instead of `system_profiler`, which took seconds on some Macs.
 - A model a server won't serve (HTTP 401-404, e.g. OpenRouter's "No endpoints found") is never judged on it: those answers don't count, and the model stops at once instead of failing every question.
 - PTA index: parsimony (tokens used) and speed (time), each 100 for the best and 15 points less per doubling, and accuracy per model, as bars: the first Results tab (where `r` lands), `tuieval pta` and the report. Server errors aren't answers, and models with few answers are left out unless picked.

{tuieval-0.2.0.dev55 → tuieval-0.2.0.dev56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev55
+Version: 0.2.0.dev56
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev55 → tuieval-0.2.0.dev56}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev55'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev55')
+__version__ = version = '0.2.0.dev56'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev56')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev55 → tuieval-0.2.0.dev56}/src/tuieval/engine.py

@@ -987,16 +987,16 @@ class Engine:
                     del per_pack[name]
         return {l: p for l, p in out.items() if p}
 
-    def estimate_seconds(self, jobs):
-        """(seconds, notes): expected run time for the waiting jobs, estimated per pack.
+    def seconds_per_answer(self, jobs, waiting_only=True):
+        """({job key: expected seconds per answer}, borrowed, guessed, thin) for the waiting jobs.
         A pack the model has answered uses its own median time there. Otherwise the median of
         other models' times on that pack, scaled by how much slower or faster this model is than
-        them on packs both have answered. notes lists what made the estimate rough ([] = solid)."""
+        them on packs both have answered. The sets say which estimates are rough (see estimate_seconds)."""
         times = self.answer_times()
         med = {l: {p: statistics.median(ts) for p, ts in pp.items()} for l, pp in times.items()}
-        total, borrowed, guessed, thin = 0.0, set(), set(), set()
+        spr_of, borrowed, guessed, thin = {}, set(), set(), set()
         for j in jobs:
-            if j.status != "waiting":
+            if j.status in ("done", "skipped", "failed") or (waiting_only and j.status != "waiting"):
                 continue
             own = med.get(j.label, {})
             ref = {p: statistics.median(v[p] for l, v in med.items() if p in v and l != j.label)
@@ -1013,8 +1013,18 @@ class Engine:
             else:
                 spr = statistics.median(own.values()) if own else DEFAULT_SECONDS_PER_REQUEST
                 guessed.add((j.key, bool(own)))
-            done = len(self._done_keys(j)) if j.merge else 0
-            total += max(0, j.total - done) * spr
+            spr_of[j.key] = spr
+        return spr_of, borrowed, guessed, thin
+
+    def estimate_seconds(self, jobs):
+        """(seconds, notes): expected run time for the waiting jobs, estimated per pack
+        (seconds_per_answer). notes lists what made the estimate rough ([] = solid)."""
+        spr_of, borrowed, guessed, thin = self.seconds_per_answer(jobs)
+        total = 0.0
+        for j in jobs:
+            if j.key in spr_of:
+                done = len(self._done_keys(j)) if j.merge else 0
+                total += max(0, j.total - done) * spr_of[j.key]
         notes = []
         if borrowed:
             factors = sorted({f for _, f, _ in borrowed if f is not None})

{tuieval-0.2.0.dev55 → tuieval-0.2.0.dev56}/src/tuieval/tui.py

@@ -1855,6 +1855,8 @@ class RunScreen(Screen):
         self.current = ""
         self.req_first = None
         self.stats = []
+        self.spr, self.rough = None, set()   # job key -> expected seconds per answer (ETA), rough ones
+        self.live = {}                        # job key -> seconds per answer in this session
         self.mem = {}
         self.outcome = ""  # "Finished" or "Cancelled" once the queue is done
         self.ended = None
@@ -2037,6 +2039,8 @@ class RunScreen(Screen):
                            + (f"  [red]copy of repeat {r['identical_to_repeat'] + 1}[/red]"
                               if r.get("identical_to_repeat") is not None else ""))
             self.stats.append(r)
+            if r.get("total_s"):
+                self.live.setdefault(job.key, []).append(r["total_s"])
             self.recent_entries[row_key] = {"model": job.label, "pack": job.pack.name, "record": r,
                                             "pack_fp": job.pack.fingerprint}
             if recent.row_count > 500:
@@ -2187,6 +2191,34 @@ class RunScreen(Screen):
                            ("note", short_note(j.note))):
             q.update_cell(j.key, col, value, update_width=True)
 
+    LIVE_ANSWERS = 5   # a pack's own times this session take over from its history after this many
+
+    def eta_seconds(self):
+        """(seconds, rough): time left, pack by pack: answers left × seconds per answer. A pack's own
+        average this session once it has LIVE_ANSWERS of them, before that its history (Engine.
+        seconds_per_answer: this model's past times there, or other models' scaled). Divided by the
+        models running side by side. rough: some packs have no history of their own yet."""
+        new = [j for j in self.jobs if self.spr is None or (j.key not in self.spr and j.status not in
+                                                           ("done", "skipped", "failed"))]
+        if new:   # at the start, and for evals added to the run
+            spr, borrowed, guessed, _ = self.app.engine.seconds_per_answer(new, waiting_only=False)
+            self.spr = {**(self.spr or {}), **spr}
+            self.rough |= {k for k, *_ in borrowed} | {k for k, _ in guessed}
+        left, rough, models = 0.0, False, set()
+        for j in self.jobs:
+            if j.status in ("done", "skipped", "failed") or j.done >= j.total:
+                continue
+            live = self.live.get(j.key, [])
+            if len(live) >= self.LIVE_ANSWERS:
+                spr = sum(live) / len(live)
+            else:
+                spr = self.spr.get(j.key, engine.DEFAULT_SECONDS_PER_REQUEST)
+                rough |= j.key in self.rough
+            left += (j.total - j.done) * spr
+            models.add(j.label)
+        lanes = max(1, min(self.parallel or self.app.engine.parallel_models(), len(models)))
+        return left / lanes, rough
+
     def update_overall(self):
         done = sum(j.done for j in self.jobs)
         passed = sum(j.passed for j in self.jobs)
@@ -2194,9 +2226,9 @@ class RunScreen(Screen):
         self.query_one("#bar", ProgressBar).update(progress=done)
         elapsed = time.time() - self.started
         eta = ""
-        done_now = sum(j.done - j.resumed for j in self.jobs if j.started)   # answers from earlier sittings
-        if self.running and done_now > 0 and self.total > done:              # took no time in this one
-            eta = f" · ETA {fmt_secs(elapsed / done_now * (self.total - done))}"
+        if self.running and self.total > done:
+            left, rough = self.eta_seconds()
+            eta = f" · ETA {'~' if rough else ''}{fmt_secs(left)}"
         pct = f"{100 * passed / (passed + failed):.0f}%" if passed + failed else "-"
         recent = self.stats[-50:]
         tps = [r["gen_tps"] for r in recent if r.get("gen_tps")]

{tuieval-0.2.0.dev55 → tuieval-0.2.0.dev56}/tests/test_tuieval.py

@@ -12,6 +12,7 @@ import sys
 import tempfile
 import textwrap
 import threading
+import types
 import time
 import unittest
 import urllib.request
@@ -199,6 +200,26 @@ class WorkspaceGraders(unittest.TestCase):
 
 
 class Units(unittest.TestCase):
+    def test_eta_goes_pack_by_pack(self):
+        from tuieval.tui import RunScreen
+        job = lambda key, label, total, done=0, status="waiting": types.SimpleNamespace(
+            key=key, label=label, total=total, done=done, status=status)
+        run = RunScreen.__new__(RunScreen)
+        run.parallel, run.rough = 1, {"local/new"}
+        run.jobs = [job("hosted/a", "hosted", 100, 100, "done"), job("local/a", "local", 200, 4, "running"),
+                    job("local/new", "local", 50)]
+        run.spr = {"local/a": 60.0, "local/new": 30.0}            # from history
+        run.live = {"local/a": [900.0, 800.0, 700.0, 1000.0]}       # its first answers are the slow ones
+        left, rough = run.eta_seconds()
+        self.assertEqual((left, rough), (196 * 60 + 50 * 30, True))  # history, not 4 slow answers: ~3.7h, not ~47h
+        run.live["local/a"].append(600.0)                            # 5 answers: its own average takes over
+        run.jobs[1].done = 5
+        self.assertEqual(run.eta_seconds()[0], 195 * 800 + 50 * 30)
+        run.parallel = 2
+        run.jobs.append(job("other/a", "other", 10))
+        run.spr["other/a"] = 10.0
+        self.assertEqual(run.eta_seconds()[0], (195 * 800 + 50 * 30 + 100) / 2)   # two models at a time
+
     def test_server_errors_never_judge_the_model(self):
         from tuieval import client
         no_endpoints = 'server returned HTTP 404: {"error":{"message":"No endpoints found for some/model."}}'