tuieval 0.2.0.dev135__tar.gz → 0.2.0.dev137__tar.gz

{tuieval-0.2.0.dev135 → tuieval-0.2.0.dev137}/CHANGELOG.md

@@ -3,7 +3,7 @@
 ## Unreleased
 
 - Answers stuck in a loop (one short unit repeated over 2000 chars, in the reasoning or the answer) are stopped as they stream and failed as `LOOP`, instead of running to max_tokens. `[defaults] loop_stop_chars` sets the length; 0 turns it off.
-- Repeats: the PTA index, `tuieval compare` and Setup show how many times each model answered each question (`×3`, `×1-3`), so single-run results stand out.
+- Repeats: the PTA index, Per question, `tuieval compare` and Setup show how many times each model answered each question (`×3`, `×1-3`), so single-run results stand out.
 - Partial credit: the PTA index shows each model's score (e.g. 5 of 6 checks passed is 83%) beside accuracy, and can be ordered by it (By score, `tuieval pta --sort score`). Why models fail splits failures into near miss, partly right and wrong; Failures shows each one's kind and checks and filters by kind; Per question cells show `✗ 5/6`. Verdicts are unchanged.
 - Results: picking a pack (PTA index or Per question) compares the models that answered all of it; the Models button can change that, and All packs goes back to the earlier pick.
 - Fix: a request that runs out of time says `time limit: timed out after N s`, not `connection error`, and `tuieval compare` counts it against the model like the run does (it was left out as an error).

{tuieval-0.2.0.dev135 → tuieval-0.2.0.dev137}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev135
+Version: 0.2.0.dev137
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev135 → tuieval-0.2.0.dev137}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev135'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev135')
+__version__ = version = '0.2.0.dev137'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev137')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev135 → tuieval-0.2.0.dev137}/src/tuieval/tui.py

@@ -431,6 +431,15 @@ def trim_names(names, width):
     return trimmed if len(set(trimmed)) == len(set(names)) else list(names)
 
 
+def repeats_tag(counts):
+    """A model's answers per question as markup, as the PTA index shows them: "×3", or "×1-3" when it varies;
+    yellow when it's ×1 (one answer decides each result, the roughest), else dim. "" for no answers."""
+    if not counts:
+        return ""
+    few, most = min(counts), max(counts)
+    return f"[{'yellow' if most == 1 else 'dim'}]×{few}{'' if few == most else f'-{most}'}[/]"
+
+
 def fit_first_column(header, rows, room, floor=24):
     """rows with the first cell trimmed in the middle (mid_trim) so a table of these columns fits room
     cells across, but never below floor cells. Cells are Rich markup; each column adds 2 cells of padding."""
@@ -3136,7 +3145,10 @@ class ResultsScreen(Screen):
             base = column_name(col.label)
             label = base + SORT_ARROWS[desc] if base == name else base
             if col.label.plain != label:
-                col.label = Text(label)
+                text = col.label.copy()   # keeps its styling, e.g. a model's ×n
+                if len(text.plain) > len(base):   # right_crop(0) would empty it
+                    text.right_crop(len(text.plain) - len(base))
+                col.label = text + SORT_ARROWS[desc] if base == name else text
                 w = cell_len(label)   # room for the arrow (a column keeps the width it was added with)
                 col.content_width = max(col.content_width, w)
                 if not col.auto_width:
@@ -3404,7 +3416,9 @@ class ResultsScreen(Screen):
         here = self.app.engine.machine().id
         table = self.query_one("#per-question", DataTable)
         table.clear(columns=True)
-        table.add_columns(*self.PQ_FIXED, *models)
+        table.add_columns(*self.PQ_FIXED, *(Text.from_markup(
+            f"{rich_escape(m)} {repeats_tag([t['cells'][m]['n'] for t in tests if m in t['cells']])}".rstrip())
+            for m in models))
         self.pq_tests, self.pq_models = {}, models
         for t in tests:
             cells = []
@@ -3427,7 +3441,8 @@ class ResultsScreen(Screen):
             "[green]bold[/green] = fewest tokens among passing answers · [red]cut[/red] = hit max_tokens · "
             "[yellow]over[/yellow] = ran past max_tokens (the provider didn't stop it) · "
             "◐ = pack unfinished (not in scores) · dim seconds = hosted, or measured on another machine; tokens compare "
-            "across machines, seconds don't. ×tokens = most / fewest tokens across models. Enter on a cell opens the "
+            "across machines, seconds don't. ×tokens = most / fewest tokens across models. ×n beside a model: answers "
+            "per question (×1: one answer decides each result, the roughest). Enter on a cell opens the "
             "answers. Click a column name to sort by it (a model's column: by its tokens).[/dim]" + ("\n[yellow]Left out, being rerun with other questions or settings: " + ", ".join(
                 f"{os.path.basename(os.path.dirname(p))} · {os.path.basename(p)[:-5]}" for p in sorted(self.superseded))
                 + ". Only the new run's answers (◐) are shown for these.[/yellow]" if self.superseded else ""))
@@ -3469,6 +3484,7 @@ class ResultsScreen(Screen):
         def hi(v, text, key, good):
             ref = best.get(key) if good == "max" else low.get(key)
             return f"[bold green]{text}[/bold green]" if v is not None and v == ref and len(rows) > 1 else text
+        tags = {m: repeats_tag([t["cells"][m]["n"] for t in shared]) for m in models}
         cells = []
         for r in rows:
             tt = r["total_secs"]
@@ -3489,7 +3505,10 @@ class ResultsScreen(Screen):
                           (f"[red]{r['cut']}[/red]" if r["cut"] else "0")
                           + (f" [yellow]+{r['over']} over[/yellow]" if r.get("over") else "")])
         table.add_columns(*header)
-        table.add_rows(fit_first_column(header, cells, (self.size.width or self.app.size.width) - 3))
+        tag_w = max(Text.from_markup(tag).cell_len + 1 for tag in tags.values())
+        cells = fit_first_column(header, cells, (self.size.width or self.app.size.width) - 3 - tag_w,
+                                 floor=24 - tag_w)   # the column's floor counts the ×n too
+        table.add_rows([[f"{rich_escape(c[0])} {tags[r['model']]}"] + c[1:] for c, r in zip(cells, rows)])
         line = f"[b]Comparison[/b] over the {len(shared)} questions all {len(models)} answered"
         if len(models) == 2:
             h = compare.head_to_head(shared, *models)
@@ -3506,7 +3525,7 @@ class ResultsScreen(Screen):
         title.update(line + "  [dim]Time / Tokens: each question's median, added up. Med tok / Med s: median "
                      "per answer. Tok/✓: tokens per answer right. Prompt t/s / Gen t/s: all tokens over all the time "
                      "they took (cached prompts left out). Passed: questions right on every repeat, then the score (partial credit, e.g. checks passed) when some failed. Fewest / Fastest: questions won; ties count for both. "
-                     "Cut off: answers that hit max_tokens.[/dim]")
+                     "Cut off: answers that hit max_tokens. ×n beside a name: answers per question.[/dim]")
 
     @staticmethod
     def pq_cell(c, best, here):

{tuieval-0.2.0.dev135 → tuieval-0.2.0.dev137}/tests/test_tuieval.py

@@ -1739,7 +1739,7 @@ class PTAIndex(unittest.TestCase):
                     print(t.row_count, str(s.query_one("#fail-count", Static).render()))
                     s.fill_per_question()
                     pq = s.query_one("#per-question", DataTable)
-                    cols = [str(c.label) for c in pq.ordered_columns]
+                    cols = [str(c.label).split(" ×")[0] for c in pq.ordered_columns]
                     print(sorted(str(pq.get_row_at(i)[cols.index("close")]).split()[1] for i in range(pq.row_count)))
             asyncio.run(go())
         """)
@@ -1801,7 +1801,7 @@ class PTAIndex(unittest.TestCase):
                         s = app.screen
                         def state():
                             pq = s.query_one("#per-question", DataTable)
-                            cols = [str(c.label) for c in pq.ordered_columns][len(s.PQ_FIXED):]
+                            cols = [str(c.label).split(" ×")[0] for c in pq.ordered_columns][len(s.PQ_FIXED):]
                             return (sorted(app.pq_models), s.query_one("#pta-pack", Select).value,
                                     str(s.query_one("#pq-models", Button).label), cols, s.query_one("#pq-shared").value)
                         s.query_one("#pq-pack", Select).value = "apps"
@@ -1955,7 +1955,7 @@ class PTAIndex(unittest.TestCase):
                             pq.move_cursor(row=0, column=col)
                             await pilot.press("enter")
                             await pilot.pause(0.3)
-                            opened[str(pq.ordered_columns[col].label.plain)] = app.screen.entries[0]["model"]
+                            opened[pq.ordered_columns[col].label.plain.split(" ×")[0]] = app.screen.entries[0]["model"]
                             app.screen.dismiss()
                             await pilot.pause(0.2)
                         print(json.dumps({"rows": rows, "bars": "P parsimony" in tri, "disagree": [both, alone], "opened": opened,
@@ -2271,6 +2271,7 @@ class FirstRun(unittest.TestCase):
         tuieval(self.ws, "add", "my-model", "--server", "local")
         code = textwrap.dedent("""
             import asyncio
+            from rich.text import Text
             from tuieval import compare
             from tuieval.tui import EvalsApp, ResultsScreen
             from textual.widgets import DataTable, TabbedContent
@@ -2288,14 +2289,56 @@ class FirstRun(unittest.TestCase):
                     app.screen.fill_pq_summary(compare.per_question(rows), models)
                     await pilot.pause(0.3)
                     t = app.screen.query_one("#pq-summary", DataTable)
-                    names = [str(t.get_row_at(i)[0]) for i in range(t.row_count)]
+                    names = [Text.from_markup(t.get_row_at(i)[0]).plain for i in range(t.row_count)]
                     print(t.row_count, t.region.height, t.virtual_size.width <= t.region.width,
-                          all(n.endswith("-q4_k_m-vllm") and "…" in n for n in names), len(set(names)))
+                          all(n.endswith("-q4_k_m-vllm ×1") and "…" in n for n in names), len(set(names)))
             asyncio.run(go())
         """)
         out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True,
                              env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""), timeout=60)
-        self.assertEqual(out.stdout.strip(), "12 13 True True 12", out.stderr[-2000:])   # every row, no scroll
+        # every row, no scroll; a trimmed name keeps its ×n
+        self.assertEqual(out.stdout.strip(), "12 13 True True 12", out.stderr[-2000:])
+
+    def test_per_question_shows_repeats(self):
+        tuieval(self.ws, "add", "my-model", "--server", "local")
+        code = textwrap.dedent("""
+            import asyncio
+            from tuieval.tui import EvalsApp, ResultsScreen
+            from textual.widgets import DataTable, TabbedContent
+            rows = [{"model": m, "suite": "p", "test": f"p: q{q}", "ok": True, "latency": 1000, "tokens": 10,
+                     "truncated": False, "gen_tps": 20.0, "prompt_tps": None, "repeat": r, "score": 1.0}
+                    for m, reps in (("once", 1), ("thrice", 3)) for q in range(2) for r in range(reps)]
+            rows += [dict(rows[-1], test="p: q2", repeat=0)]   # one more question, answered once
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(160, 40)) as pilot:
+                    await pilot.pause()
+                    app.push_screen(ResultsScreen())
+                    await pilot.pause(0.5)
+                    s = app.screen
+                    s.rows, s.partial_rows, s.superseded = rows, [], set()
+                    s.query_one(TabbedContent).active = "tab-per-question"
+                    await pilot.pause(0.3)
+                    s.fill_per_question()
+                    await pilot.pause(0.3)
+                    pq = s.query_one("#per-question", DataTable)
+                    heads = [c.label for c in pq.ordered_columns][len(s.PQ_FIXED):]
+                    print([h.plain for h in heads], [str(h.spans[0].style) for h in heads])
+                    app.result_sorts["per-question"] = ("thrice ×1-3", True)   # as a click on its name does
+                    s.sort_table(pq)
+                    head = pq.ordered_columns[-1].label
+                    print(head.plain, str(head.spans[0].style))
+                    t = s.query_one("#pq-summary", DataTable)
+                    print([t.get_row_at(i)[0] for i in range(t.row_count)])
+            asyncio.run(go())
+        """)
+        out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True,
+                             env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""), timeout=60)
+        self.assertEqual(out.stdout.strip().splitlines(), [
+            "['once ×1', 'thrice ×1-3'] ['yellow', 'dim']",   # over the questions shown
+            "thrice ×1-3 ▼ dim",                               # sorting keeps the ×n's colour
+            "['once [yellow]×1[/]', 'thrice [dim]×3[/]']",     # Comparison: only the questions both answered
+        ], out.stderr[-2000:])
 
     def test_results_open_on_pta_index(self):
         tuieval(self.ws, "add", "my-model", "--server", "local")