tuieval 0.2.0.dev105__tar.gz → 0.2.0.dev108__tar.gz

{tuieval-0.2.0.dev105 → tuieval-0.2.0.dev108}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- The Models heading shows how many are selected, naming any the filter hides; Start asks first when some are hidden.
+- Results → Failures: a Models button filters the list (the same pick as the other tabs), with the number of failures shown.
 - Fix: an answer followed by a stray `</think>` (or a stray code fence after it) was graded as empty; the answer before the tag is graded now. The code grader also reads a doubled ```` ```python ```` fence as one.
 - Run screen: `x` removes the highlighted pack from the run going if it hasn't started (the reverse of adding it); a model left with nothing isn't started.
 - `k` (skip model) asks first, naming the model. Yes/no dialogs start on No; `y` answers yes, `n` or Esc no.

{tuieval-0.2.0.dev105 → tuieval-0.2.0.dev108}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev105
+Version: 0.2.0.dev108
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev105 → tuieval-0.2.0.dev108}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev105'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev105')
+__version__ = version = '0.2.0.dev108'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev108')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev105 → tuieval-0.2.0.dev108}/src/tuieval/tui.py

@@ -454,7 +454,8 @@ with the model's reasoning and answer as they stream, then recent results with t
                          Results open here.
   Production readiness   one verdict per model and use case; the table says why and what to run next
   Per question           every question, model by model (filters: pack, level, Models disagree)
-  Failures               every wrong answer with the grader's reason; enter opens it in full
+  Failures               every wrong answer with the grader's reason (filter: Models); enter opens it
+                         in full
 
 Click a column name to sort a table by it; click it again to reverse. Empty cells stay last.
 
@@ -775,10 +776,7 @@ class SetupScreen(Screen):
             self.verdicts = {}
             self.app.log(f"verdicts failed: {err!r}")
         codes = group_codes(e)
-        self.query_one("#models-legend", Label).update(
-            "[b]Models[/b]  [dim][green]✓[/green] pass  [red]✗[/red] fail  [yellow]?[/yellow] inconclusive  "
-            "grey = earlier  ·  x hides[/dim]\n[dim]"
-            + "  ".join(f"{c} {g}" for g, c in codes.items()) + "[/dim]")
+        self.legend_codes = codes
         n_hidden = sum(m["label"] in hidden for m in e.cfg["models"])
         self.query_one("#show-hidden", Checkbox).label = f"Show hidden ({n_hidden})"
         sections = {}
@@ -818,8 +816,27 @@ class SetupScreen(Screen):
             at = values.index(keep) if keep in values else min(keep_at or 0, models.option_count - 1)
             models.highlighted = min((i for i in enabled if i >= at), default=enabled[-1])
         self.show_detail()
+        self.update_legend()
         self.update_estimate()
 
+    def unseen_models(self):
+        """Ticked models the list doesn't show right now (the filter leaves them out)."""
+        models = self.query_one("#models", SelectionList)
+        visible = {models.get_option_at_index(i).value for i in range(models.option_count)}
+        return [m["label"] for m in self.app.engine.cfg["models"]
+                if m["label"] in self.selected_models and m["label"] not in visible]
+
+    def update_legend(self):
+        """The Models heading: how many are ticked (and which ones the filter hides), then the key."""
+        n, unseen = len(self.selected_models), self.unseen_models()
+        count = f"[b]{n}[/b] selected" if n else "[dim]none selected[/dim]"
+        if unseen:
+            count += f" [yellow]({len(unseen)} not shown: {rich_escape(', '.join(unseen))})[/yellow]"
+        self.query_one("#models-legend", Label).update(
+            f"[b]Models[/b]  {count}  [dim][green]✓[/green] pass  [red]✗[/red] fail  [yellow]?[/yellow] "
+            "inconclusive  grey = earlier  ·  x hides[/dim]\n[dim]"
+            + "  ".join(f"{c} {g}" for g, c in getattr(self, "legend_codes", {}).items()) + "[/dim]")
+
     @on(SelectionList.SelectionHighlighted, "#models")
     def show_detail(self, *_):
         """Everything about the highlighted model: verdicts in full, serving, per-pack results."""
@@ -893,6 +910,7 @@ class SetupScreen(Screen):
             return
         visible = {models.get_option_at_index(i).value for i in range(models.option_count)}
         self.selected_models = (self.selected_models - visible) | set(models.selected)
+        self.update_legend()
         self.update_estimate()
 
     def tier(self):
@@ -1109,10 +1127,20 @@ class SetupScreen(Screen):
                 self.notify(f"Nothing to run: {item.problem}.", severity="warning")
                 return
             self.app.start_or_queue(item)
-        if rows:
-            self.app.push_screen(RetestScreen(rows, force, rerun), go)
+        def asked(yes=True):
+            if not yes:
+                return
+            if rows:
+                self.app.push_screen(RetestScreen(rows, force, rerun), go)
+            else:
+                go({})
+        unseen = self.unseen_models()
+        if unseen:   # ticks the filter hides are easy to forget: name them before anything starts
+            self.app.push_screen(ConfirmScreen(
+                f"Start {len(labels)} model(s)? {len(unseen)} of them the filter hides:\n  "
+                + rich_escape("\n  ".join(unseen) + f"\n\nAll selected: {', '.join(labels)}")), asked)
         else:
-            go({})
+            asked()
 
     def action_tune(self):
         e = self.app.engine
@@ -2825,6 +2853,9 @@ class ResultsScreen(Screen):
                 yield Static(id="pq-legend")
                 yield DataTable(id="per-question", zebra_stripes=True, cursor_type="cell")
             with TabPane("Failures", id="tab-failures"):
+                with Horizontal(id="fail-controls"):
+                    yield Button("Models: all", id="fail-models")
+                    yield Static(id="fail-count")
                 yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
                 yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
         yield Footer()
@@ -2902,12 +2933,7 @@ class ResultsScreen(Screen):
         info.update(f"{len(paths)} {'Smoke ' if smoke else ''}result files in {os.path.relpath(results_dir)}/" + banner
                     + "".join(f"\n[yellow]⚠ {n}[/yellow]" for n in notes))
         self.rows, self.infos = rows, infos
-        f = self.query_one("#failures", DataTable)
-        f.add_columns("Model", "Level", "Test", "Reason")
-        # compare.failures lists the failed rows in order, so they line up with fail_rows
-        self.fail_rows = [r for r in rows if not r["ok"]]
-        for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
-            f.add_row(model, diff_badge(level), rich_escape(test), rich_escape(reason[:130]), key=str(i))
+        self.fill_failures()
         self.fill_per_question()
         self.fill_pta()
         self.fill_readiness(results_dir)
@@ -2916,6 +2942,24 @@ class ResultsScreen(Screen):
             self.fill_insights()
         self.sort_tables()
 
+    def fill_failures(self):
+        """Every wrong answer, of the models picked (Models: shared with the other tabs)."""
+        f = self.query_one("#failures", DataTable)
+        f.clear(columns=True)
+        f.add_columns("Model", "Level", "Test", "Reason")
+        every = {r["model"] for r in self.rows}
+        chosen = {m for m in self.app.pq_models if m in every}
+        rows = [r for r in self.rows if not chosen or r["model"] in chosen]
+        # compare.failures lists the failed rows in order, so they line up with fail_rows
+        self.fail_rows = [r for r in rows if not r["ok"]]
+        for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
+            f.add_row(model, diff_badge(level), rich_escape(test), rich_escape(reason[:130]), key=str(i))
+        self.query_one("#fail-models", Button).label = f"Models: {len(chosen)} picked" if chosen else "Models: all"
+        n = len(self.fail_rows)
+        self.query_one("#fail-count", Static).update(
+            f"{n} failure{'s' * (n != 1)}" + (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
+        self.sort_table(f)
+
     @on(DataTable.RowSelected, "#failures")
     def failure_selected(self, event):
         # in the order shown, so [ and ] step through them as sorted
@@ -2979,6 +3023,7 @@ class ResultsScreen(Screen):
     @on(Button.Pressed, "#pq-models")
     @on(Button.Pressed, "#pta-models")
     @on(Button.Pressed, "#ins-models")
+    @on(Button.Pressed, "#fail-models")
     def pick_models(self):
         counts = {}
         for t in getattr(self, "pq_all_tests", []):
@@ -2997,6 +3042,7 @@ class ResultsScreen(Screen):
             # after the picker has closed: a table rebuilt under a modal keeps its header-only widths
             self.call_after_refresh(self.fill_per_question)
             self.call_after_refresh(self.fill_pta)
+            self.call_after_refresh(self.fill_failures)
             self.insights_stale = True
             if self.query_one(TabbedContent).active == "tab-insights":
                 self.call_after_refresh(self.fill_insights)
@@ -3397,6 +3443,9 @@ class EvalsApp(App):
     #pq-controls Checkbox { width: auto; }
     #pq-controls Select { width: 22; margin-right: 1; }
     #pq-models { margin-right: 1; min-width: 20; }
+    #fail-controls { height: 3; }
+    #fail-models { margin-right: 1; min-width: 20; }
+    #fail-count { width: auto; padding: 1 1; }
     #pq-compare-title { padding: 0 1; height: auto; }
     #pq-summary { height: auto; }   /* every picked model, no scrolling: the tab scrolls instead */
     #pq-legend { padding: 0 1; height: auto; }

{tuieval-0.2.0.dev105 → tuieval-0.2.0.dev108}/tests/test_tuieval.py

@@ -1130,6 +1130,47 @@ class AddToRun(unittest.TestCase):
         self.assertEqual(fewer, 4)                     # its 4 questions are no longer counted
         self.assertEqual(out["running"], [False, True])
 
+    def test_tui_counts_ticked_models_and_names_filtered_ones_before_start(self):
+        code = textwrap.dedent("""
+            import asyncio, json
+            from tuieval.tui import EvalsApp, ConfirmScreen
+            from textual.widgets import SelectionList, Input, Label
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(160, 50)) as pilot:
+                    await pilot.pause()
+                    s, out = app.screen, {}
+                    legend = lambda: str(s.query_one("#models-legend", Label).render())
+                    out["none"] = legend()
+                    s.query_one("#suites", SelectionList).select("apps")
+                    models = s.query_one("#models", SelectionList)
+                    models.select("a")
+                    models.select("b")
+                    await pilot.pause()
+                    out["two"] = legend()
+                    s.query_one("#filter", Input).value = "zzz-nothing"
+                    await pilot.pause()
+                    out["filtered"] = legend()
+                    s.action_start()
+                    await pilot.pause()
+                    out["asked"] = isinstance(app.screen, ConfirmScreen) and str(app.screen.question)
+                    await pilot.press("n")
+                    await pilot.pause()
+                    out["cancelled"] = app.screen is s and not app.is_busy()
+                    print(json.dumps(out))
+            asyncio.run(go())
+        """)
+        p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=600,
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""))
+        out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
+        self.assertIsInstance(out, dict, out)
+        self.assertIn("none selected", out["none"])
+        self.assertIn("2 selected", out["two"])
+        self.assertNotIn("not shown", out["two"])
+        self.assertIn("2 not shown: a, b", out["filtered"])
+        self.assertIn("All selected: a, b", out["asked"])
+        self.assertTrue(out["cancelled"])
+
     def test_tui_asks_before_skipping_a_model(self):
         path = os.path.join(self.ws, "models.toml")
         write(path, read(path).replace('"--delay", "0.2"', '"--delay", "3"'))   # still running while it's asked
@@ -1571,6 +1612,17 @@ class PTAIndex(unittest.TestCase):
                         app.screen.query_one(TabbedContent).active = "tab-failures"
                         await pilot.pause(0.3)
                         f = app.screen.query_one("#failures", DataTable)
+                        count = lambda: str(app.screen.query_one("#fail-count", Static).render())
+                        fails = {"all": [f.row_count, count()]}
+                        for m in ("good", "hosted"):                            # Models: only that model's failures
+                            app.pq_models = {m}
+                            app.screen.fill_failures()
+                            await pilot.pause(0.2)
+                            fails[m] = [f.row_count, count(), str(app.screen.query_one("#fail-models").label),
+                                        sorted({str(f.get_row_at(r)[0]) for r in range(f.row_count)})]
+                        app.pq_models = set()
+                        app.screen.fill_failures()
+                        await pilot.pause(0.2)
                         await click_header(f, "Test")
                         sorts["tests"], _ = await click_header(f, "Test")
                         sorts["tests"] = [str(f.get_row_at(r)[2]) for r in range(f.row_count)]
@@ -1581,7 +1633,7 @@ class PTAIndex(unittest.TestCase):
                         s = app.screen
                         sorts["opened"] = s.entries[s.index]["record"]["test"]
                         print(json.dumps({"rows": rows, "bars": "P parsimony" in tri, "disagree": [both, alone],
-                                          "tab": tab, "sorts": sorts}))
+                                          "tab": tab, "sorts": sorts, "fails": fails}))
                 asyncio.run(go())
             """)
             p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=300,
@@ -1596,6 +1648,10 @@ class PTAIndex(unittest.TestCase):
             self.assertEqual((s["up"], s["down"], s["kept"]), (["hosted", "good"], ["good", "hosted"], ["good", "hosted"]))
             self.assertIn("answers right ▼", s["labels"])
             self.assertEqual((s["tests"][0], s["opened"]), ("apps: d", "d"))
+            fails = got["fails"]
+            self.assertEqual(fails["all"], [4, "4 failures"])
+            self.assertEqual(fails["good"], [0, "0 failures of 1 model", "Models: 1 picked", []])
+            self.assertEqual(fails["hosted"], [4, "4 failures of 1 model", "Models: 1 picked", ["hosted"]])
 
 
 class Unavailable(unittest.TestCase):