tuieval 0.2.0.dev81__py3-none-any.whl → 0.2.0.dev87__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev81'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev81')
+__version__ = version = '0.2.0.dev87'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev87')
 
 __commit_id__ = commit_id = None

tuieval/tui.py

@@ -1550,6 +1550,13 @@ def run_title(tier, models, packs):
             f"{', '.join(packs) if len(packs) <= 2 else f'{len(packs)} packs'}")
 
 
+def short_keys(keys, n=4):
+    """A few "model/pack" pairs as "model · pack", the rest counted."""
+    keys = sorted(keys)
+    shown = ", ".join(k.replace("/", " · ", 1) for k in keys[:n])
+    return rich_escape(shown + (f" and {len(keys) - n} more" if len(keys) > n else ""))
+
+
 class QueuedRun:
     """A run waiting its turn. It's planned again when it starts, so it picks up what the runs before
     it finished; it never sets aside answers that weren't confirmed when it was queued. retest: the
@@ -1557,13 +1564,29 @@ class QueuedRun:
     kind = "Run"
 
     def __init__(self, sel, retest, leave, accepted, eng):
-        self.sel, self.retest, self.leave, self.accepted = sel, retest, leave, accepted
+        self.sel, self.retest, self.leave, self.accepted = sel, retest, set(leave), accepted
         self.queued = time.time()
+        self.replan(eng)
+
+    def replan(self, eng):
         self.jobs, self.problem = self.plan(eng)
         jobs = self.jobs or []
-        self.title = run_title(sel["tier"], {j.label for j in jobs}, {j.pack.label for j in jobs})
+        self.title = run_title(self.sel["tier"], {j.label for j in jobs}, {j.pack.label for j in jobs})
         self.est = eng.estimate_seconds(jobs)[0]
 
+    def keys(self):
+        """The "model/pack" pairs it covers."""
+        return {f"{m}/{p}" for m in self.sel["models"] for p in self.sel["packs"]} - self.leave
+
+    def deliberate(self):
+        """Retest or picked questions: asked again on purpose, never treated as a duplicate."""
+        return bool(self.sel.get("force") or self.sel.get("tests"))
+
+    def drop(self, keys, eng):
+        """Leave these pairs out (they're already running or queued)."""
+        self.leave |= set(keys)
+        self.replan(eng)
+
     def tests(self, eng):
         """The picked tests that are still in their packs (a pack edited while queued drops the rest)."""
         out = {}
@@ -2183,6 +2206,7 @@ class RunScreen(Screen):
         yield Footer()
 
     def on_mount(self):
+        self._queue_follow, self._queue_y = True, None   # follow_queue
         q = self.query_one("#queue", DataTable)
         for label, key in (("Model", "model"), ("Eval", "suite"), ("Status", "status"), ("Progress", "progress"),
                            ("✓", "pass"), ("✗", "fail"), ("Time", "time"), ("Note", "note")):
@@ -2500,6 +2524,25 @@ class RunScreen(Screen):
                 self.update_job(j)
         self.update_overall()
         self.refresh_current()
+        self.follow_queue()
+
+    def follow_queue(self):
+        """Keep the running model's row in view in the queue, unless you scrolled the queue yourself;
+        scrolling back to where the running row shows picks the following up again."""
+        q = self.query_one("#queue", DataTable)
+        keys = [r.key.value for r in q.ordered_rows]
+        now = next((i for i, k in enumerate(keys)
+                    if any(j.key == k and j.status in ("loading", "running") for j in self.jobs)), None)
+        if now is None:
+            return
+        if self._queue_y is not None and q.scroll_y != self._queue_y:   # you scrolled since we last did
+            seen = q.scroll_y <= now < q.scroll_y + max(1, q.scrollable_content_region.height - 1)
+            self._queue_follow = seen
+        if self._queue_follow:
+            q.scroll_to(y=max(0, now - 1), animate=False)
+            self._queue_y = max(0, min(now - 1, q.max_scroll_y))
+        else:
+            self._queue_y = q.scroll_y
 
     # -- actions
     def action_skip_model(self):
@@ -3236,8 +3279,32 @@ class EvalsApp(App):
         what = s.live_text() if s else "The last run is still stopping its server"
         return what + (f" (+{len(self.queue)} queued)" if self.queue else "")
 
+    def taken(self, tier):
+        """The "model/pack" pairs of this tier already in the run going (not finished) or in the queue."""
+        out = set()
+        run = self.active_session
+        if isinstance(run, RunScreen) and run.running and run.jobs and run.jobs[0].tier == tier:
+            out |= {j.key for j in run.jobs if j.status in ("waiting", "loading", "running")}
+        for q in self.queue:
+            if q.kind == "Run" and q.sel["tier"] == tier:
+                out |= q.keys()
+        return out
+
     # -- the queue: Start or t while something runs adds to it; each item starts when the one before ends
     def start_or_queue(self, item):
+        skipped = set()
+        if self.is_busy() and item.kind == "Run" and not item.deliberate():
+            mine = item.keys()
+            skipped = mine & self.taken(item.sel["tier"])
+            if skipped == mine:
+                self.notify(f"Already running or queued, so nothing was added: {short_keys(skipped)}.", timeout=10)
+                return
+            if skipped:
+                item.drop(skipped, self.engine)
+                if not item.jobs:
+                    self.notify(f"Nothing new to run: {item.problem}. Already running or queued: "
+                                f"{short_keys(skipped)}.", timeout=10)
+                    return
         if not self.is_busy():
             screen, problem = item.make(self.engine)
             if problem:
@@ -3256,11 +3323,11 @@ class EvalsApp(App):
                     self.add_to_run(run, item)
                 elif choice == "queue":
                     self.enqueue(item)
-            what = f"{', '.join(item.sel['models'][:4])}{' …' if len(item.sel['models']) > 4 else ''} · " \
-                   f"{', '.join(item.sel['packs'][:4])}{' …' if len(item.sel['packs']) > 4 else ''}"
+            what = short_keys(item.keys(), 6)
+            skip = (f"\n\nLeft out, already running or queued: {short_keys(skipped)}." if skipped else "")
             self.push_screen(ChoiceScreen(
                 f"A run is going: {run.live_text()}.\n\nAdd {what} to it? Its models run after the ones "
-                "already in it.\nOr queue it as a separate run that starts when this one ends.",
+                f"already in it.\nOr queue it as a separate run that starts when this one ends.{skip}",
                 [("add", "Add to this run", "success"), ("queue", f"Queue it as #{len(self.queue) + 1}", "primary"),
                  ("keep", "Cancel", "default")]), picked)
             return
@@ -3270,6 +3337,9 @@ class EvalsApp(App):
         jobs, problem = item.plan(self.engine)
         added, why = self.engine.add_jobs(jobs) if jobs else ([], problem)
         if not added:
+            if why and why.startswith("everything picked is already in the run"):
+                self.notify(f"Not added: {why}.", timeout=10)   # queueing it again would only duplicate it
+                return
             self.notify(f"Not added: {why}. Queued it instead.", severity="warning", timeout=10)
             self.enqueue(item)
             return
@@ -3281,6 +3351,8 @@ class EvalsApp(App):
     def enqueue(self, item):
         ahead = self.busy_text()
         self.queue.append(item)
+        if not self.active_session:   # the last run already ended (its server may still be stopping):
+            self.set_timer(0.3, self.start_next)   # nothing else would start the queue
         self.notify(f"Queued #{len(self.queue)}: {item.title}. It starts by itself after: {ahead}. "
                     "w shows the queue.", timeout=10)
         self.refresh_live()

{tuieval-0.2.0.dev81.dist-info → tuieval-0.2.0.dev87.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev81
+Version: 0.2.0.dev87
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev81.dist-info → tuieval-0.2.0.dev87.dist-info}/RECORD

@@ -1,6 +1,6 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=wvLQcBRh9Ke_k0afcu8VifHptYL1UWBFOCSFcnpeLi0,535
+tuieval/_version.py,sha256=MbDAskHZqL7Ly_g4fmQSqzeTJS3q2IGRhFiHFeJp5zA,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=VwiaU1F9bZtU8C-gEowJq58kPeYQY4t6SY_HwrV-qgc,4702
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
@@ -18,7 +18,7 @@ tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=FU3bvu3jt9ZWg3YLsXNINbdeClXMioyc4zQ4yEhmBQs,47855
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=bGAy0cP55rWkg9Cj3iAsSqR3tNM67Cj3G3nfvCitN8A,173351
+tuieval/tui.py,sha256=ZE9FEOr84Vx2YULSiBpaZM4DyOo0iIkuKlH870K88I8,176810
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -47,8 +47,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev81.dist-info/METADATA,sha256=362fng3M9sOKj7AGpBf-ebNRTQgTjW84h1nfrn3YeKg,14580
-tuieval-0.2.0.dev81.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev81.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev81.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev81.dist-info/RECORD,,
+tuieval-0.2.0.dev87.dist-info/METADATA,sha256=XHHaef6pqjG5DaNsjztrZJ9K_ZPhGi63ONwA_J8XFqg,14580
+tuieval-0.2.0.dev87.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev87.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev87.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev87.dist-info/RECORD,,