tuieval 0.2.0.dev139__tar.gz → 0.2.0.dev141__tar.gz

{tuieval-0.2.0.dev139 → tuieval-0.2.0.dev141}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Per question → Comparison: a **Pass rate** column beside Passed, the share of answers right with repeats included (shown when some model has repeats).
 - Repeats (Setup, `tuieval run --repeat N`) add N rounds to the saved answers: ×1 run with 2 gives ×3, not ×2. Retest, a blank count and presets still mean the total.
 - Answers stuck in a loop (one short unit repeated over 2000 chars, in the reasoning or the answer) are stopped as they stream and failed as `LOOP`, instead of running to max_tokens. `[defaults] loop_stop_chars` sets the length; 0 turns it off.
 - Repeats: the PTA index, Per question, `tuieval compare` and Setup show how many times each model answered each question (`×3`, `×1-3`), so single-run results stand out.

{tuieval-0.2.0.dev139 → tuieval-0.2.0.dev141}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev139
+Version: 0.2.0.dev141
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev139 → tuieval-0.2.0.dev141}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev139'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev139')
+__version__ = version = '0.2.0.dev141'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev141')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev139 → tuieval-0.2.0.dev141}/src/tuieval/compare.py

@@ -420,7 +420,8 @@ def complete_models(rows, suite, test_ids=None):
 def compare_models(tests, models):
     """Per model over per_question entries that all `models` answered: passes, token and time
     totals/medians (totals add each question's median over repeats, so repeats don't inflate them), questions where it used the fewest tokens / was fastest (among passing
-    answers for tokens; ties count for each), answers cut off, mean score (partial credit)."""
+    answers for tokens; ties count for each), answers cut off, mean score (partial credit),
+    pass rate (answers right with repeats, each question weighing the same)."""
     out = []
     for m in models:
         cs = [t["cells"][m] for t in tests]
@@ -438,6 +439,7 @@ def compare_models(tests, models):
         rs = [r for c in cs for r in c["rows"]]
         out.append({"model": m, "n": len(cs), "passed": passed, "total_tokens": total,
                     "score": statistics.fmean(c["score"] for c in cs) if cs else None,
+                    "pass_rate": statistics.fmean(c["passed"] / c["n"] for c in cs) if cs else None,
                     "avg_pp_tps": _rate([(r.get("prompt_tokens"), r["prompt_tps"]) for r in rs]),
                     "avg_gen_tps": _rate([(r["tokens"], r["gen_tps"]) for r in rs]),
                     "med_tokens": median(c["tokens"] for c in cs), "med_secs": median(c["secs"] for c in cs),

{tuieval-0.2.0.dev139 → tuieval-0.2.0.dev141}/src/tuieval/tui.py

@@ -3487,10 +3487,10 @@ class ResultsScreen(Screen):
         self.pq_summary_args = (tests, models)    # to refit on resize
         rows = compare.compare_models(shared, models)
         # short headings so every column fits; the line above the table says what each one is
-        header = ("Model", "Time", "Passed", "Med tok", "Tokens", "Tok/✓", "Med s", "Prompt t/s", "Gen t/s",
-                  "Fewest", "Fastest", "Cut off")
+        header = ("Model", "Time", "Passed", "Pass rate", "Med tok", "Tokens", "Tok/✓", "Med s", "Prompt t/s",
+                  "Gen t/s", "Fewest", "Fastest", "Cut off")
         best = {k: max((r[k] for r in rows if r[k] is not None), default=None)
-                for k in ("passed", "fewest", "fastest", "avg_pp_tps", "avg_gen_tps")}
+                for k in ("passed", "pass_rate", "fewest", "fastest", "avg_pp_tps", "avg_gen_tps")}
         low = {k: min(r[k] for r in rows if r[k] is not None) if any(r[k] is not None for r in rows) else None
                for k in ("med_tokens", "total_tokens", "tokens_per_pass", "med_secs", "total_secs")}
 
@@ -3505,6 +3505,7 @@ class ResultsScreen(Screen):
                           hi(tt, ("-" if not tt else f"{tt:.1f}s" if tt < 60 else fmt_secs(tt)), "total_secs", "min"),
                           hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max")
                           + (f" [dim]{r['score']:.0%}[/dim]" if r["passed"] < r["n"] and r["score"] else ""),
+                          hi(r["pass_rate"], f"{r['pass_rate']:.0%}", "pass_rate", "max"),
                           hi(r["med_tokens"], f"{r['med_tokens']:,.0f}" if r["med_tokens"] else "-", "med_tokens", "min"),
                           hi(r["total_tokens"], f"{r['total_tokens']:,.0f}", "total_tokens", "min"),
                           hi(r["tokens_per_pass"], f"{r['tokens_per_pass']:,.0f}" if r["tokens_per_pass"] else "-",
@@ -3517,6 +3518,9 @@ class ResultsScreen(Screen):
                           hi(r["fastest"], str(r["fastest"]), "fastest", "max"),
                           (f"[red]{r['cut']}[/red]" if r["cut"] else "0")
                           + (f" [yellow]+{r['over']} over[/yellow]" if r.get("over") else "")])
+        if all(t["cells"][m]["n"] == 1 for t in shared for m in models):   # one answer each: Passed says it all
+            i = header.index("Pass rate")
+            header, cells = header[:i] + header[i + 1:], [c[:i] + c[i + 1:] for c in cells]
         table.add_columns(*header)
         tag_w = max(Text.from_markup(tag).cell_len + 1 for tag in tags.values())
         cells = fit_first_column(header, cells, (self.size.width or self.app.size.width) - 3 - tag_w,
@@ -3537,7 +3541,7 @@ class ResultsScreen(Screen):
                      "shared the machine's speed: compare their times with care.[/yellow]")
         title.update(line + "  [dim]Time / Tokens: each question's median, added up. Med tok / Med s: median "
                      "per answer. Tok/✓: tokens per answer right. Prompt t/s / Gen t/s: all tokens over all the time "
-                     "they took (cached prompts left out). Passed: questions right on every repeat, then the score (partial credit, e.g. checks passed) when some failed. Fewest / Fastest: questions won; ties count for both. "
+                     "they took (cached prompts left out). Passed: questions right on every repeat, then the score (partial credit, e.g. checks passed) when some failed. Pass rate (with repeats): answers right, repeats included (each question weighs the same). Fewest / Fastest: questions won; ties count for both. "
                      "Cut off: answers that hit max_tokens. ×n beside a name: answers per question.[/dim]")
 
     @staticmethod

{tuieval-0.2.0.dev139 → tuieval-0.2.0.dev141}/tests/test_tuieval.py

@@ -2451,6 +2451,52 @@ class FirstRun(unittest.TestCase):
             "['once [yellow]×1[/]', 'thrice [dim]×3[/]']",     # Comparison: only the questions both answered
         ], out.stderr[-2000:])
 
+    def test_pass_rate_counts_every_answer(self):
+        from tuieval import compare
+        row = {"suite": "p", "latency": 1000, "tokens": 10, "truncated": False, "gen_tps": 20.0, "prompt_tps": None}
+        oks = {"a": {"q0": [True, True], "q1": [True, False], "q2": [False, False, False, True]},   # ×2, ×2, ×4
+               "b": {"q0": [True], "q1": [False], "q2": [False]},
+               "c": {"q0": [False], "q1": [False], "q2": [False]}}
+        rows = [dict(row, model=m, test=f"p: {q}", ok=ok, repeat=r)
+                for m, qs in oks.items() for q, v in qs.items() for r, ok in enumerate(v)]
+        out = {r["model"]: r for r in compare.compare_models(compare.per_question(rows), list(oks))}
+        self.assertEqual(out["a"]["passed"], 1)                            # strict: right on every repeat
+        self.assertAlmostEqual(out["a"]["pass_rate"], (1 + 0.5 + 0.25) / 3)   # each question weighs the same
+        self.assertAlmostEqual(out["b"]["pass_rate"], 1 / 3)               # ×1: the same as passed / questions
+        self.assertEqual(out["c"]["pass_rate"], 0)
+
+    def test_comparison_shows_pass_rate(self):
+        tuieval(self.ws, "add", "my-model", "--server", "local")
+        code = textwrap.dedent("""
+            import asyncio
+            from rich.text import Text
+            from tuieval import compare
+            from tuieval.tui import EvalsApp, ResultsScreen
+            from textual.widgets import DataTable, TabbedContent
+            oks = {"steady": [[True, True], [True, True]], "shaky": [[True, True], [True, False]]}
+            rows = [{"model": m, "suite": "p", "test": f"p: q{q}", "ok": ok, "latency": 1000, "tokens": 10,
+                     "truncated": False, "gen_tps": 20.0, "prompt_tps": None, "repeat": r}
+                    for m, qs in oks.items() for q, v in enumerate(qs) for r, ok in enumerate(v)]
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(160, 40)) as pilot:
+                    await pilot.pause()
+                    app.push_screen(ResultsScreen())
+                    await pilot.pause(0.5)
+                    app.screen.query_one(TabbedContent).active = "tab-per-question"
+                    await pilot.pause(0.3)
+                    app.screen.fill_pq_summary(compare.per_question(rows), list(oks))
+                    await pilot.pause(0.3)
+                    t = app.screen.query_one("#pq-summary", DataTable)
+                    i = [str(c.label) for c in t.ordered_columns].index("Pass rate")
+                    print(str(t.ordered_columns[i - 1].label), [t.get_row_at(n)[i] for n in range(t.row_count)])
+            asyncio.run(go())
+        """)
+        out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True,
+                             env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""), timeout=60)
+        # right after Passed; the best in bold green
+        self.assertEqual(out.stdout.strip(), "Passed ['[bold green]100%[/bold green]', '75%']", out.stderr[-2000:])
+
     def test_results_open_on_pta_index(self):
         tuieval(self.ws, "add", "my-model", "--server", "local")
         code = textwrap.dedent("""