tuieval 0.2.0.dev81__tar.gz → 0.2.0.dev86__tar.gz

{tuieval-0.2.0.dev81 → tuieval-0.2.0.dev86}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Start while a run is going skips models and packs already running or queued (it says which); picking only those adds nothing. Retest and picked questions still go through.
+- Fix: something queued just as a run ended (while its server was stopping) now starts instead of waiting forever.
 - Hosted models (OpenRouter and other APIs) run beside local ones instead of waiting for a slot: up to `parallel_hosted` at a time (default 4, `--parallel-hosted N`). Local models can be added to a hosted-only run.
 - Early stop: `tuieval run --stop-early`, **Stop a pack once it fails** in Setup, or `stop_early = true` in a pack.toml ends a pack once its FAIL is certain (a critical failure, or an accuracy or truncation bar out of reach), asking first the questions models fail most. Verdicts also say FAIL when a partial run can no longer pass.
 - `tuieval compare --reference <model>`: every other model paired against one on the questions both answered, per pack and overall: the difference with a 95% CI, better/tie/worse, critical failures and tokens vs the reference. `--packs` limits any compare to some packs.

{tuieval-0.2.0.dev81 → tuieval-0.2.0.dev86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev81
+Version: 0.2.0.dev86
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev81 → tuieval-0.2.0.dev86}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev81'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev81')
+__version__ = version = '0.2.0.dev86'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev86')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev81 → tuieval-0.2.0.dev86}/src/tuieval/tui.py

@@ -1550,6 +1550,13 @@ def run_title(tier, models, packs):
             f"{', '.join(packs) if len(packs) <= 2 else f'{len(packs)} packs'}")
 
 
+def short_keys(keys, n=4):
+    """A few "model/pack" pairs as "model · pack", the rest counted."""
+    keys = sorted(keys)
+    shown = ", ".join(k.replace("/", " · ", 1) for k in keys[:n])
+    return rich_escape(shown + (f" and {len(keys) - n} more" if len(keys) > n else ""))
+
+
 class QueuedRun:
     """A run waiting its turn. It's planned again when it starts, so it picks up what the runs before
     it finished; it never sets aside answers that weren't confirmed when it was queued. retest: the
@@ -1557,13 +1564,29 @@ class QueuedRun:
     kind = "Run"
 
     def __init__(self, sel, retest, leave, accepted, eng):
-        self.sel, self.retest, self.leave, self.accepted = sel, retest, leave, accepted
+        self.sel, self.retest, self.leave, self.accepted = sel, retest, set(leave), accepted
         self.queued = time.time()
+        self.replan(eng)
+
+    def replan(self, eng):
         self.jobs, self.problem = self.plan(eng)
         jobs = self.jobs or []
-        self.title = run_title(sel["tier"], {j.label for j in jobs}, {j.pack.label for j in jobs})
+        self.title = run_title(self.sel["tier"], {j.label for j in jobs}, {j.pack.label for j in jobs})
         self.est = eng.estimate_seconds(jobs)[0]
 
+    def keys(self):
+        """The "model/pack" pairs it covers."""
+        return {f"{m}/{p}" for m in self.sel["models"] for p in self.sel["packs"]} - self.leave
+
+    def deliberate(self):
+        """Retest or picked questions: asked again on purpose, never treated as a duplicate."""
+        return bool(self.sel.get("force") or self.sel.get("tests"))
+
+    def drop(self, keys, eng):
+        """Leave these pairs out (they're already running or queued)."""
+        self.leave |= set(keys)
+        self.replan(eng)
+
     def tests(self, eng):
         """The picked tests that are still in their packs (a pack edited while queued drops the rest)."""
         out = {}
@@ -3236,8 +3259,32 @@ class EvalsApp(App):
         what = s.live_text() if s else "The last run is still stopping its server"
         return what + (f" (+{len(self.queue)} queued)" if self.queue else "")
 
+    def taken(self, tier):
+        """The "model/pack" pairs of this tier already in the run going (not finished) or in the queue."""
+        out = set()
+        run = self.active_session
+        if isinstance(run, RunScreen) and run.running and run.jobs and run.jobs[0].tier == tier:
+            out |= {j.key for j in run.jobs if j.status in ("waiting", "loading", "running")}
+        for q in self.queue:
+            if q.kind == "Run" and q.sel["tier"] == tier:
+                out |= q.keys()
+        return out
+
     # -- the queue: Start or t while something runs adds to it; each item starts when the one before ends
     def start_or_queue(self, item):
+        skipped = set()
+        if self.is_busy() and item.kind == "Run" and not item.deliberate():
+            mine = item.keys()
+            skipped = mine & self.taken(item.sel["tier"])
+            if skipped == mine:
+                self.notify(f"Already running or queued, so nothing was added: {short_keys(skipped)}.", timeout=10)
+                return
+            if skipped:
+                item.drop(skipped, self.engine)
+                if not item.jobs:
+                    self.notify(f"Nothing new to run: {item.problem}. Already running or queued: "
+                                f"{short_keys(skipped)}.", timeout=10)
+                    return
         if not self.is_busy():
             screen, problem = item.make(self.engine)
             if problem:
@@ -3256,11 +3303,11 @@ class EvalsApp(App):
                     self.add_to_run(run, item)
                 elif choice == "queue":
                     self.enqueue(item)
-            what = f"{', '.join(item.sel['models'][:4])}{' …' if len(item.sel['models']) > 4 else ''} · " \
-                   f"{', '.join(item.sel['packs'][:4])}{' …' if len(item.sel['packs']) > 4 else ''}"
+            what = short_keys(item.keys(), 6)
+            skip = (f"\n\nLeft out, already running or queued: {short_keys(skipped)}." if skipped else "")
             self.push_screen(ChoiceScreen(
                 f"A run is going: {run.live_text()}.\n\nAdd {what} to it? Its models run after the ones "
-                "already in it.\nOr queue it as a separate run that starts when this one ends.",
+                f"already in it.\nOr queue it as a separate run that starts when this one ends.{skip}",
                 [("add", "Add to this run", "success"), ("queue", f"Queue it as #{len(self.queue) + 1}", "primary"),
                  ("keep", "Cancel", "default")]), picked)
             return
@@ -3270,6 +3317,9 @@ class EvalsApp(App):
         jobs, problem = item.plan(self.engine)
         added, why = self.engine.add_jobs(jobs) if jobs else ([], problem)
         if not added:
+            if why and why.startswith("everything picked is already in the run"):
+                self.notify(f"Not added: {why}.", timeout=10)   # queueing it again would only duplicate it
+                return
             self.notify(f"Not added: {why}. Queued it instead.", severity="warning", timeout=10)
             self.enqueue(item)
             return
@@ -3281,6 +3331,8 @@ class EvalsApp(App):
     def enqueue(self, item):
         ahead = self.busy_text()
         self.queue.append(item)
+        if not self.active_session:   # the last run already ended (its server may still be stopping):
+            self.set_timer(0.3, self.start_next)   # nothing else would start the queue
         self.notify(f"Queued #{len(self.queue)}: {item.title}. It starts by itself after: {ahead}. "
                     "w shows the queue.", timeout=10)
         self.refresh_live()

{tuieval-0.2.0.dev81 → tuieval-0.2.0.dev86}/tests/test_tuieval.py

@@ -1007,6 +1007,10 @@ class AddToRun(unittest.TestCase):
             import asyncio, json, time
             from tuieval.tui import EvalsApp, RunScreen, SetupScreen, ChoiceScreen
             from textual.widgets import SelectionList
+            async def setup(app, pilot):   # a dialog closing can take a moment on a slow machine
+                deadline = time.time() + 30
+                while not isinstance(app.screen, SetupScreen) and time.time() < deadline:
+                    await pilot.pause(0.1)
             async def go():
                 app = EvalsApp({})
                 async with app.run_test(size=(160, 50)) as pilot:
@@ -1033,7 +1037,7 @@ class AddToRun(unittest.TestCase):
                     await pilot.pause()
                     asked = isinstance(app.screen, ChoiceScreen)
                     app.screen.dismiss("add")
-                    await pilot.pause()
+                    await setup(app, pilot)
                     app.screen.query_one("#suites", SelectionList).deselect("apps")
                     app.screen.query_one("#suites", SelectionList).select("other")
                     app.screen.selected_models = {"a"}
@@ -1041,7 +1045,7 @@ class AddToRun(unittest.TestCase):
                     app.screen.action_start()
                     await pilot.pause()
                     app.screen.dismiss("queue")
-                    await pilot.pause()
+                    await setup(app, pilot)
                     app.screen.query_one("#tier-smoke").value = True    # another tier can't join: queued
                     await pilot.pause()
                     app.screen.action_start()
@@ -1074,6 +1078,76 @@ class AddToRun(unittest.TestCase):
         self.assertEqual(out["second"], [["a/other", "done"]])                  # queued: its own run after
 
 
+    def test_tui_never_queues_the_same_thing_twice(self):
+        """Start while a run is going never adds or queues a model and pack already running or queued."""
+        code = textwrap.dedent("""
+            import asyncio, json, time
+            from tuieval.tui import EvalsApp, RunScreen, SetupScreen, ChoiceScreen
+            from textual.widgets import SelectionList
+            async def pick(app, pilot, models, packs=("apps",)):
+                deadline = time.time() + 30
+                while not isinstance(app.screen, SetupScreen) and time.time() < deadline:
+                    await pilot.pause(0.1)
+                s = app.screen
+                for p in ("apps", "other"):
+                    (s.query_one("#suites", SelectionList).select if p in packs
+                     else s.query_one("#suites", SelectionList).deselect)(p)
+                s.selected_models = set(models)
+                s.refresh_models()
+                await pilot.pause()
+                s.action_start()
+                await pilot.pause()
+                return app.screen
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(160, 50)) as pilot:
+                    await pilot.pause()
+                    app.screen.query_one("#tier-certify").value = True
+                    await pick(app, pilot, ["a"])
+                    deadline = time.time() + 180
+                    while time.time() < deadline:
+                        await pilot.pause(0.1)
+                        if isinstance(app.screen, RunScreen) and app.screen.counts()[0]:
+                            break
+                    run = app.screen
+                    await pilot.press("n")
+                    out = {}
+                    s = await pick(app, pilot, ["a"])                 # the same again: nothing happens
+                    out["same"] = [isinstance(s, ChoiceScreen), len(app.queue)]
+                    s = await pick(app, pilot, ["a", "b"])            # partly new: only b is offered
+                    out["partly"] = [isinstance(s, ChoiceScreen), "Left out" in str(getattr(s, "question", ""))]
+                    s.dismiss("queue")
+                    await pilot.pause()
+                    out["queued"] = [sorted(q.keys()) for q in app.queue]
+                    s = await pick(app, pilot, ["b"])                 # already queued: nothing happens
+                    out["again"] = [isinstance(s, ChoiceScreen), len(app.queue)]
+                    s = await pick(app, pilot, ["a"], ("other",))     # new: offered, queued or started
+                    out["new"] = (isinstance(s, ChoiceScreen) or any("a/other" in q.keys() for q in app.queue)
+                                  or any(j.key == "a/other" for x in app.sessions for j in x.jobs))
+                    if isinstance(s, ChoiceScreen):
+                        s.dismiss("queue")
+                    await pilot.pause()
+                    deadline = time.time() + 300   # the queued runs start one after another: wait for all three
+                    while time.time() < deadline:
+                        await pilot.pause(0.2)
+                        if len(app.sessions) == 3 and not app.queue and not any(x.running for x in app.sessions):
+                            break
+                    out["sessions"] = sorted((j.key, j.status) for x in app.sessions for j in x.jobs)
+                    print(json.dumps(out))
+            asyncio.run(go())
+        """)
+        p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=600,
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""))
+        out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
+        self.assertIsInstance(out, dict, out)
+        self.assertEqual(out["same"], [False, 0])
+        self.assertEqual(out["partly"], [True, True])
+        self.assertEqual(out["queued"], [["b/apps"]])
+        self.assertEqual(out["again"], [False, 1])
+        self.assertTrue(out["new"])
+        self.assertEqual(out["sessions"], [["a/apps", "done"], ["a/other", "done"], ["b/apps", "done"]])
+
+
 class PTAIndex(unittest.TestCase):
     """The PTA index: parsimony from tokens and speed from time, each relative to the best on a log
     scale, and accuracy, all over the questions every compared model answered."""