tuieval 0.2.0.dev107__tar.gz → 0.2.0.dev108__tar.gz

{tuieval-0.2.0.dev107 → tuieval-0.2.0.dev108}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- The Models heading shows how many are selected, naming any the filter hides; Start asks first when some are hidden.
 - Results → Failures: a Models button filters the list (the same pick as the other tabs), with the number of failures shown.
 - Fix: an answer followed by a stray `</think>` (or a stray code fence after it) was graded as empty; the answer before the tag is graded now. The code grader also reads a doubled ```` ```python ```` fence as one.
 - Run screen: `x` removes the highlighted pack from the run going if it hasn't started (the reverse of adding it); a model left with nothing isn't started.

{tuieval-0.2.0.dev107 → tuieval-0.2.0.dev108}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev107
+Version: 0.2.0.dev108
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev107 → tuieval-0.2.0.dev108}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev107'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev107')
+__version__ = version = '0.2.0.dev108'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev108')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev107 → tuieval-0.2.0.dev108}/src/tuieval/tui.py

@@ -776,10 +776,7 @@ class SetupScreen(Screen):
             self.verdicts = {}
             self.app.log(f"verdicts failed: {err!r}")
         codes = group_codes(e)
-        self.query_one("#models-legend", Label).update(
-            "[b]Models[/b]  [dim][green]✓[/green] pass  [red]✗[/red] fail  [yellow]?[/yellow] inconclusive  "
-            "grey = earlier  ·  x hides[/dim]\n[dim]"
-            + "  ".join(f"{c} {g}" for g, c in codes.items()) + "[/dim]")
+        self.legend_codes = codes
         n_hidden = sum(m["label"] in hidden for m in e.cfg["models"])
         self.query_one("#show-hidden", Checkbox).label = f"Show hidden ({n_hidden})"
         sections = {}
@@ -819,8 +816,27 @@ class SetupScreen(Screen):
             at = values.index(keep) if keep in values else min(keep_at or 0, models.option_count - 1)
             models.highlighted = min((i for i in enabled if i >= at), default=enabled[-1])
         self.show_detail()
+        self.update_legend()
         self.update_estimate()
 
+    def unseen_models(self):
+        """Ticked models the list doesn't show right now (the filter leaves them out)."""
+        models = self.query_one("#models", SelectionList)
+        visible = {models.get_option_at_index(i).value for i in range(models.option_count)}
+        return [m["label"] for m in self.app.engine.cfg["models"]
+                if m["label"] in self.selected_models and m["label"] not in visible]
+
+    def update_legend(self):
+        """The Models heading: how many are ticked (and which ones the filter hides), then the key."""
+        n, unseen = len(self.selected_models), self.unseen_models()
+        count = f"[b]{n}[/b] selected" if n else "[dim]none selected[/dim]"
+        if unseen:
+            count += f" [yellow]({len(unseen)} not shown: {rich_escape(', '.join(unseen))})[/yellow]"
+        self.query_one("#models-legend", Label).update(
+            f"[b]Models[/b]  {count}  [dim][green]✓[/green] pass  [red]✗[/red] fail  [yellow]?[/yellow] "
+            "inconclusive  grey = earlier  ·  x hides[/dim]\n[dim]"
+            + "  ".join(f"{c} {g}" for g, c in getattr(self, "legend_codes", {}).items()) + "[/dim]")
+
     @on(SelectionList.SelectionHighlighted, "#models")
     def show_detail(self, *_):
         """Everything about the highlighted model: verdicts in full, serving, per-pack results."""
@@ -894,6 +910,7 @@ class SetupScreen(Screen):
             return
         visible = {models.get_option_at_index(i).value for i in range(models.option_count)}
         self.selected_models = (self.selected_models - visible) | set(models.selected)
+        self.update_legend()
         self.update_estimate()
 
     def tier(self):
@@ -1110,10 +1127,20 @@ class SetupScreen(Screen):
                 self.notify(f"Nothing to run: {item.problem}.", severity="warning")
                 return
             self.app.start_or_queue(item)
-        if rows:
-            self.app.push_screen(RetestScreen(rows, force, rerun), go)
+        def asked(yes=True):
+            if not yes:
+                return
+            if rows:
+                self.app.push_screen(RetestScreen(rows, force, rerun), go)
+            else:
+                go({})
+        unseen = self.unseen_models()
+        if unseen:   # ticks the filter hides are easy to forget: name them before anything starts
+            self.app.push_screen(ConfirmScreen(
+                f"Start {len(labels)} model(s)? {len(unseen)} of them the filter hides:\n  "
+                + rich_escape("\n  ".join(unseen) + f"\n\nAll selected: {', '.join(labels)}")), asked)
         else:
-            go({})
+            asked()
 
     def action_tune(self):
         e = self.app.engine

{tuieval-0.2.0.dev107 → tuieval-0.2.0.dev108}/tests/test_tuieval.py

@@ -1130,6 +1130,47 @@ class AddToRun(unittest.TestCase):
         self.assertEqual(fewer, 4)                     # its 4 questions are no longer counted
         self.assertEqual(out["running"], [False, True])
 
+    def test_tui_counts_ticked_models_and_names_filtered_ones_before_start(self):
+        code = textwrap.dedent("""
+            import asyncio, json
+            from tuieval.tui import EvalsApp, ConfirmScreen
+            from textual.widgets import SelectionList, Input, Label
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(160, 50)) as pilot:
+                    await pilot.pause()
+                    s, out = app.screen, {}
+                    legend = lambda: str(s.query_one("#models-legend", Label).render())
+                    out["none"] = legend()
+                    s.query_one("#suites", SelectionList).select("apps")
+                    models = s.query_one("#models", SelectionList)
+                    models.select("a")
+                    models.select("b")
+                    await pilot.pause()
+                    out["two"] = legend()
+                    s.query_one("#filter", Input).value = "zzz-nothing"
+                    await pilot.pause()
+                    out["filtered"] = legend()
+                    s.action_start()
+                    await pilot.pause()
+                    out["asked"] = isinstance(app.screen, ConfirmScreen) and str(app.screen.question)
+                    await pilot.press("n")
+                    await pilot.pause()
+                    out["cancelled"] = app.screen is s and not app.is_busy()
+                    print(json.dumps(out))
+            asyncio.run(go())
+        """)
+        p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=600,
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""))
+        out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
+        self.assertIsInstance(out, dict, out)
+        self.assertIn("none selected", out["none"])
+        self.assertIn("2 selected", out["two"])
+        self.assertNotIn("not shown", out["two"])
+        self.assertIn("2 not shown: a, b", out["filtered"])
+        self.assertIn("All selected: a, b", out["asked"])
+        self.assertTrue(out["cancelled"])
+
     def test_tui_asks_before_skipping_a_model(self):
         path = os.path.join(self.ws, "models.toml")
         write(path, read(path).replace('"--delay", "0.2"', '"--delay", "3"'))   # still running while it's asked