tuieval 0.2.0.dev74__tar.gz → 0.2.0.dev75__tar.gz

{tuieval-0.2.0.dev74 → tuieval-0.2.0.dev75}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix: a question description or grader reason containing `[` no longer crashes the run screen.
 - Retest asks per pack before it starts: from zero, continue or leave out. It now really starts from zero, and replaced smoke results go to history/.
 - Retest: the setup checkbox "Rerun existing results" is now "Retest (ask again, even if already answered)", and `tuieval run --retest` is the same (`--force` still works).
 - `tuieval logs` follows the server log of the model running now and moves on with the run; `logs live` shows each answer as it finishes, `--list` every log.

{tuieval-0.2.0.dev74 → tuieval-0.2.0.dev75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev74
+Version: 0.2.0.dev75
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev74 → tuieval-0.2.0.dev75}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev74'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev74')
+__version__ = version = '0.2.0.dev75'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev75')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev74 → tuieval-0.2.0.dev75}/src/tuieval/tui.py

@@ -2239,9 +2239,10 @@ class RunScreen(Screen):
             self.req_started, self.req_tokens, self.req_first = time.time(), 0, None
             job = d["job"]
             rep_note = f" · repeat {d['repeat'] + 1}/{job.repeat}" if job.repeat > 1 else ""
-            self.current = (f"[b]{job.label}[/b] · {job.pack.label} · test {job.done + 1}/{job.total}{rep_note}"
+            self.current = (f"[b]{rich_escape(job.label)}[/b] · {rich_escape(job.pack.label)} · test "
+                            f"{job.done + 1}/{job.total}{rep_note}"
                             f"{'  [magenta]🖼 image[/magenta]' if d['has_image'] else ''}\n"
-                            f"{diff_badge(d.get('difficulty', 'unrated'))}  {d['test']}")
+                            f"{diff_badge(d.get('difficulty', 'unrated'))}  {rich_escape(d['test'])}")
         elif kind == "request_done":
             if self.watched(kind, d):
                 self.flush_streams()
@@ -2251,12 +2252,12 @@ class RunScreen(Screen):
             # follow new rows unless you're browsing an older one
             at_end = not recent.has_focus or recent.cursor_row >= recent.row_count - 1
             calls = f"  [cyan]→ {r['tool_calls'][0]['name']}()[/cyan]" if r.get("tool_calls") else ""
-            row_key = recent.add_row(mark, diff_badge(r.get("difficulty", "unrated")), r["description"][:50], job.label,
+            row_key = recent.add_row(mark, diff_badge(r.get("difficulty", "unrated")), rich_escape(r["description"][:50]), job.label,
                            str(r.get("completion_tokens") or ""),
                            (f"{r['ttft_s']:.1f}" + (" ⟲" if r.get("cached_tokens") else ""))
                            if r.get("ttft_s") is not None else "",
                            f"{r['gen_tps']:.0f}" if r.get("gen_tps") else "", f"{r['total_s']:.0f}",
-                           r["reason"][:90] + calls
+                           rich_escape(r["reason"][:90]) + calls
                            + (f"  [red]copy of repeat {r['identical_to_repeat'] + 1}[/red]"
                               if r.get("identical_to_repeat") is not None else ""))
             self.stats.append(r)
@@ -2364,7 +2365,7 @@ class RunScreen(Screen):
             self.event("[bold yellow]REUSE[/bold yellow] " + msg)
             self.notify(msg, severity="warning", timeout=20)
         elif kind == "identical_answer":
-            msg = (f"{d['label']} · {d['pack']} · {d['test']}: repeat {d['repeat'] + 1} is an exact copy of repeat "
+            msg = (f"{d['label']} · {d['pack']} · {rich_escape(d['test'])}: repeat {d['repeat'] + 1} is an exact copy of repeat "
                    f"{d['same'] + 1}, so the server may be returning cached answers rather than fresh ones.")
             self.event("[bold red]COPY[/bold red] " + msg)
             self.notify(msg, severity="error", timeout=20)
@@ -2711,7 +2712,7 @@ class ResultsScreen(Screen):
         # compare.failures lists the failed rows in order, so they line up with fail_rows
         self.fail_rows = [r for r in rows if not r["ok"]]
         for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
-            f.add_row(model, diff_badge(level), test, rich_escape(reason[:130]), key=str(i))
+            f.add_row(model, diff_badge(level), rich_escape(test), rich_escape(reason[:130]), key=str(i))
         self.fill_per_question()
         self.fill_pta()
         self.fill_readiness(results_dir)

{tuieval-0.2.0.dev74 → tuieval-0.2.0.dev75}/tests/test_tuieval.py

@@ -735,6 +735,56 @@ class Parallel(unittest.TestCase):
         self.assertEqual(sum(r["pass"] for r in self.results("b")["results"]), 6)
 
 
+class MarkupInText(unittest.TestCase):
+    """Brackets in a description or a grader's reason are shown as text: an unmatched
+    "[" once crashed the run screen (it swallowed the "[dim]" of the line after it)."""
+
+    def test_run_screen_shows_brackets(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            ws = os.path.join(tmp, "ws")
+            tuieval(tmp, "init", ws)
+            d = os.path.join(ws, "packs", "br")
+            os.makedirs(d)
+            write(os.path.join(d, "pack.toml"), 'label = "Brackets"\n[certify]\nrepeat = 1\n[gate]\nmin_accuracy = 0.1\n')
+            write(os.path.join(d, "tests.yaml"),
+                  "- {id: a, description: 'notional in [t - 10 s, t), excluding ties', input: q a, expected: 1, "
+                  "reference: 'ANSWER: 1', wrong: ['ANSWER: [2'], difficulty: hard}\n"
+                  "- {id: b, description: 'window [a, b) then [/dim] text', input: q b, expected: 1, "
+                  "reference: 'ANSWER: 1', wrong: ['ANSWER: [2'], difficulty: hard}\n")
+            mock = [sys.executable, os.path.join(HERE, "mock_server.py"), "--port", "{port}", "--packs",
+                    os.path.join(ws, "packs"), "--model", "{served_name}", "--mode", "wrong", "--delay", "0.2"]
+            with open(os.path.join(ws, "models.toml"), "a") as f:
+                f.write(f'\n[servers.mk]\ncmd = {json.dumps(mock)}\nport = {free_port()}\n'
+                        '\n[[models]]\nlabel = "m"\nserver = "mk"\nmodel = "m"\n')
+            code = textwrap.dedent("""
+                import asyncio, json, time
+                from tuieval.tui import EvalsApp, RunScreen
+                from textual.widgets import SelectionList
+                async def go():
+                    app = EvalsApp({})
+                    async with app.run_test(size=(160, 50)) as pilot:
+                        await pilot.pause()
+                        s = app.screen
+                        s.query_one("#suites", SelectionList).select("br")
+                        s.selected_models = {"m"}
+                        s.refresh_models()
+                        s.query_one("#tier-certify").value = True
+                        await pilot.pause()
+                        s.action_start()
+                        deadline = time.time() + 240
+                        while time.time() < deadline:
+                            await pilot.pause(0.2)
+                            if isinstance(app.screen, RunScreen) and not app.screen.running:
+                                break
+                        print(json.dumps([(j.key, j.status, j.done) for j in app.screen.jobs]))
+                asyncio.run(go())
+            """)
+            p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=480,
+                               env=dict(os.environ, TUIEVAL_HOME=ws, TUIEVAL_DETECT_PORTS=""))
+            self.assertNotIn("MarkupError", p.stderr + p.stdout)
+            self.assertEqual(json.loads(p.stdout.strip().splitlines()[-1]), [["m/br", "done", 2]], p.stderr[-2000:])
+
+
 class AddToRun(unittest.TestCase):
     """More evals while a run is going: added to it (they run after its other jobs) or queued after it."""