tuieval 0.2.0.dev108__tar.gz → 0.2.0.dev112__tar.gz

{tuieval-0.2.0.dev108 → tuieval-0.2.0.dev112}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Copy reasoning: `y` in the answer view copies its full reasoning, `a` the answer; `y` on the run screen copies the reasoning streamed so far. Uses pbcopy, wl-copy, xclip, xsel or clip, else the terminal (OSC 52).
+- PTA index: click a model's name to remove it from view (every Results tab follows); click it under the bars to bring it back, or show all.
 - The Models heading shows how many are selected, naming any the filter hides; Start asks first when some are hidden.
 - Results → Failures: a Models button filters the list (the same pick as the other tabs), with the number of failures shown.
 - Fix: an answer followed by a stray `</think>` (or a stray code fence after it) was graded as empty; the answer before the tag is graded now. The code grader also reads a doubled ```` ```python ```` fence as one.

{tuieval-0.2.0.dev108 → tuieval-0.2.0.dev112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev108
+Version: 0.2.0.dev112
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -66,7 +66,7 @@ While a run goes, you watch each question's reasoning and answer stream in:
 
 ![tuieval's run screen with a model's reasoning streaming in for a coding question](https://raw.githubusercontent.com/ashe-wb/tuieval/main/docs/images/tui-run-reasoning.png)
 
-Afterwards, every answer keeps its reasoning. Enter on a result opens it with the grade and each check, the full reasoning and the answer; `[` and `]` step through the others:
+Afterwards, every answer keeps its reasoning. Enter on a result opens it with the grade and each check, the full reasoning and the answer; `[` and `]` step through the others, `y` copies the reasoning and `a` the answer (on the run screen, `y` copies the reasoning streamed so far):
 
 ![one answer in full: the grade and its checks, the model's reasoning and its code](https://raw.githubusercontent.com/ashe-wb/tuieval/main/docs/images/tui-answer-reasoning.png)
 

{tuieval-0.2.0.dev108 → tuieval-0.2.0.dev112}/README.md

@@ -40,7 +40,7 @@ While a run goes, you watch each question's reasoning and answer stream in:
 
 ![tuieval's run screen with a model's reasoning streaming in for a coding question](https://raw.githubusercontent.com/ashe-wb/tuieval/main/docs/images/tui-run-reasoning.png)
 
-Afterwards, every answer keeps its reasoning. Enter on a result opens it with the grade and each check, the full reasoning and the answer; `[` and `]` step through the others:
+Afterwards, every answer keeps its reasoning. Enter on a result opens it with the grade and each check, the full reasoning and the answer; `[` and `]` step through the others, `y` copies the reasoning and `a` the answer (on the run screen, `y` copies the reasoning streamed so far):
 
 ![one answer in full: the grade and its checks, the model's reasoning and its code](https://raw.githubusercontent.com/ashe-wb/tuieval/main/docs/images/tui-answer-reasoning.png)
 

{tuieval-0.2.0.dev108 → tuieval-0.2.0.dev112}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev108'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev108')
+__version__ = version = '0.2.0.dev112'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev112')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev108 → tuieval-0.2.0.dev112}/src/tuieval/pta.py

@@ -117,12 +117,13 @@ def _blocks(width, v, most):
     return max(1, min(width, round(width * v / most)))
 
 
-def bars(result, width=20, markup=True):
+def bars(result, width=20, markup=True, click=None):
     """Lines with a row per model and a bar each for P, T and A: the at-a-glance view. The P and T bars
     are proportional to tokens and time (the longest is the model that used most; shorter is better),
     with how many times the leanest / fastest model's beside them; the A bar is the share of answers
     right (longer is better, full only when every answer passed), then the share cut off at max_tokens in
-    orange (▓ in plain text) and "(n% truncated)" beside it. markup=False gives plain text."""
+    orange (▓ in plain text) and "(n% truncated)" beside it. markup=False gives plain text; click (an
+    action, e.g. "screen.remove") makes each model's name a link calling it with the row's index."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
@@ -133,7 +134,7 @@ def bars(result, width=20, markup=True):
     sub = pad + "".join(f"{better:<{cell_w}}" for _, _, better, *_ in LETTERS)
     lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip(), f"[dim]{sub.rstrip()}[/dim]" if markup else sub.rstrip()]
     most = {m: max((x.get(m) or 0 for x in result["models"]), default=0) for _, _, _, m, _, _ in LETTERS}
-    for x in result["models"]:
+    for row, x in enumerate(result["models"]):
         cells = []
         for k, _, _, measure, shown, color in LETTERS:
             if k == "A":
@@ -155,7 +156,11 @@ def bars(result, width=20, markup=True):
                    if markup else bar + cut_bar + rest)
             cells.append(f"{bar} {num:>6}" + " " * max(0, cell_w - width - 1 - max(6, len(num))))
         name = f"{x['model']:<{name_w}}"
-        lines.append((f"{rich_escape(name) if markup else name}  " + "".join(cells)).rstrip())
+        if markup:
+            pad_w = name_w - len(x["model"])
+            name = (f"[@click={click}({row})]{rich_escape(x['model'])}[/]" + " " * pad_w if click
+                    else rich_escape(name))
+        lines.append((f"{name}  " + "".join(cells)).rstrip())
     return lines
 
 

{tuieval-0.2.0.dev108 → tuieval-0.2.0.dev112}/src/tuieval/tui.py

@@ -24,6 +24,8 @@ import glob
 import json
 import os
 import re
+import shutil
+import subprocess
 import threading
 import time
 
@@ -289,6 +291,37 @@ class StreamView(TextArea):
         self.call_after_refresh(lambda: self.scroll_to(y=max(0, self.max_scroll_y - from_end), animate=False))
 
 
+CLIPBOARD_TOOLS = (["pbcopy"], ["wl-copy"], ["xclip", "-selection", "clipboard"], ["xsel", "--clipboard", "--input"],
+                   ["clip"])
+
+
+def copy_text(app, text):
+    """Put text on the system clipboard with the first tool here that takes it (pbcopy, wl-copy, xclip, xsel,
+    clip). Without one, the terminal is asked to (OSC 52), which not every terminal allows. True if a tool did."""
+    for cmd in CLIPBOARD_TOOLS:
+        if not shutil.which(cmd[0]):
+            continue
+        try:
+            subprocess.run(cmd, input=text.encode("utf-16" if cmd[0] == "clip" else "utf-8"),
+                           capture_output=True, check=True, timeout=10)
+            return True
+        except (OSError, subprocess.SubprocessError):
+            continue      # e.g. xclip without a display: try the next
+    app.copy_to_clipboard(text)
+    return False
+
+
+def notify_copied(app, what, text, note=""):
+    if not text:
+        app.notify(f"No {what} to copy.", severity="warning")
+    elif copy_text(app, text):
+        app.notify(f"Copied {what}{note}: {len(text):,} chars.")
+    else:
+        app.notify(f"Sent {what}{note} ({len(text):,} chars) to the terminal's clipboard. If nothing pastes, "
+                   "the terminal doesn't allow it: install a clipboard tool (wl-copy, xclip or xsel).",
+                   severity="warning", timeout=10)
+
+
 # ------------------------------------------------------------------ answer detail
 def answer_entry(row):
     """An answer for AnswerScreen from a compare row (None when the row has no stored answer)."""
@@ -451,7 +484,7 @@ with the model's reasoning and answer as they stream, then recent results with t
   PTA index              parsimony (tokens), time and accuracy per model, a bar each: tokens and time
                          as long as they took (shorter is better; 2.4× = 2.4 times the best), accuracy
                          longer is better. Per-answer numbers, tok/s, TTFT and memory in the table.
-                         Results open here.
+                         Results open here. Click a model's name to remove it from view (every tab).
   Production readiness   one verdict per model and use case; the table says why and what to run next
   Per question           every question, model by model (filters: pack, level, Models disagree)
   Failures               every wrong answer with the grader's reason (filter: Models); enter opens it
@@ -511,11 +544,14 @@ class AnswerScreen(ModalScreen):
         Binding("q", "close", "Close", show=False, priority=True),
         Binding("left_square_bracket", "step(-1)", "Previous", priority=True),
         Binding("right_square_bracket", "step(1)", "Next", priority=True),
+        Binding("y", "copy('reasoning')", "Copy reasoning", priority=True),
+        Binding("a", "copy('answer')", "Copy answer", priority=True),
     ]
 
     def __init__(self, entries, index=0, what="answer"):
         super().__init__()
         self.entries, self.index, self.what = entries, index, what
+        self.texts = {}       # what's shown: "reasoning", "answer" -> text, for the copy keys
 
     def check_action(self, action, parameters):
         return len(self.entries) > 1 if action == "step" else True
@@ -550,6 +586,9 @@ class AnswerScreen(ModalScreen):
         self.index = (self.index + delta) % len(self.entries)
         self.show()
 
+    def action_copy(self, what):
+        notify_copied(self.app, what, self.texts.get(what, ""))
+
     def show(self):
         e = self.app.engine
         entry = self.entries[self.index]
@@ -608,6 +647,7 @@ class AnswerScreen(ModalScreen):
             answer += "\n\n=== TOOL CALLS ===\n" + json.dumps(rec["tool_calls"], indent=2)
         for wid, text in (("#answer-reasoning", reasoning), ("#answer-answer", answer or "(empty)")):
             self.query_one(wid, TextArea).load_text(text)
+        self.texts = {"reasoning": reasoning, "answer": answer}
 
         question, checked = "(test not found)", "(test not found)"
         if test:
@@ -2163,6 +2203,7 @@ class RunScreen(Screen):
         Binding("c", "cancel_all", "Cancel all", priority=True),
         Binding("f", "toggle_follow", "Pause scroll", priority=True),
         Binding("v", "watch_next", "Next model", priority=True),
+        Binding("y", "copy_reasoning", "Copy reasoning", priority=True),
         Binding("n", "new_run", "More evals", priority=True),
         Binding("r", "results", "Results", priority=True),
         Binding("w", "runs", "Runs", priority=True),
@@ -2182,6 +2223,8 @@ class RunScreen(Screen):
         self.follow = True
         self.buf = {"reasoning": [], "answer": []}
         self.buf_lock = threading.Lock()
+        self.streamed = {"reasoning": [], "answer": []}   # the watched answer so far, untrimmed (for y)
+        self.streamed_whole = False                        # from its start (not joined midway with v)
         self.req_live = {}                    # model label -> its request now: phase, ~tokens, started, last token
         self.started = time.time()
         self.req_started = None
@@ -2344,6 +2387,7 @@ class RunScreen(Screen):
                 v.clear()
         for stream, text in parts.items():
             if text:
+                self.streamed[stream].append(text)
                 self.query_one(f"#{stream}", StreamView).append(text, self.follow)
                 self.req_tokens += max(1, len(text) // 4)
                 if self.req_first is None and self.req_started:
@@ -2364,6 +2408,7 @@ class RunScreen(Screen):
             self.flush_streams()
             self.query_one("#reasoning", StreamView).clear()
             self.query_one("#answer", StreamView).clear()
+            self.streamed, self.streamed_whole = {"reasoning": [], "answer": []}, True
             self.req_started, self.req_tokens, self.req_first = time.time(), 0, None
             job = d["job"]
             rep_note = f" · repeat {d['repeat'] + 1}/{job.repeat}" if job.repeat > 1 else ""
@@ -2731,6 +2776,7 @@ class RunScreen(Screen):
                 v.clear()
         self.query_one("#reasoning", StreamView).clear()
         self.query_one("#answer", StreamView).clear()
+        self.streamed, self.streamed_whole = {"reasoning": [], "answer": []}, False
         self.req_started = None
         self.current = f"Streaming [b]{self.watch}[/b] from its next answer…"
         self.refresh_current()
@@ -2748,6 +2794,12 @@ class RunScreen(Screen):
         self.follow = not self.follow
         self.update_overall()
 
+    def action_copy_reasoning(self):
+        """The streaming answer's reasoning so far, all of it (the pane drops the oldest text when it's long)."""
+        self.flush_streams()
+        notify_copied(self.app, "reasoning", "".join(self.streamed["reasoning"]),
+                      "" if self.streamed_whole else " (from midway: streaming joined after it began)")
+
     @on(DataTable.RowSelected, "#recent")
     def recent_selected(self, event):
         keys = [k for k in event.data_table.rows if k in self.recent_entries]
@@ -3032,22 +3084,51 @@ class ResultsScreen(Screen):
         models = sorted(counts, key=lambda m: (-counts[m], m))
 
         def picked(chosen):
-            if chosen is None:
-                return
-            first = chosen and not self.app.pq_models
-            self.app.pq_models = chosen  # kept for this session, so reopening Results keeps it
-            if first:  # a fair comparison needs the same questions: turn it on (you can untick it)
-                with self.query_one("#pq-shared", Checkbox).prevent(Checkbox.Changed):
-                    self.query_one("#pq-shared", Checkbox).value = True
-            # after the picker has closed: a table rebuilt under a modal keeps its header-only widths
-            self.call_after_refresh(self.fill_per_question)
-            self.call_after_refresh(self.fill_pta)
-            self.call_after_refresh(self.fill_failures)
-            self.insights_stale = True
-            if self.query_one(TabbedContent).active == "tab-insights":
-                self.call_after_refresh(self.fill_insights)
+            if chosen is not None:
+                self.set_models(chosen)
         self.app.push_screen(ModelPickScreen(models, counts, self.app.pq_models), picked)
 
+    def set_models(self, chosen):
+        """The models every Results tab compares (none: all of them)."""
+        first = chosen and not self.app.pq_models
+        self.app.pq_models = chosen  # kept for this session, so reopening Results keeps it
+        if first:  # a fair comparison needs the same questions: turn it on (you can untick it)
+            with self.query_one("#pq-shared", Checkbox).prevent(Checkbox.Changed):
+                self.query_one("#pq-shared", Checkbox).value = True
+        # after a picker has closed: a table rebuilt under a modal keeps its header-only widths
+        self.call_after_refresh(self.fill_per_question)
+        self.call_after_refresh(self.fill_pta)
+        self.call_after_refresh(self.fill_failures)
+        self.insights_stale = True
+        if self.query_one(TabbedContent).active == "tab-insights":
+            self.call_after_refresh(self.fill_insights)
+
+    def action_pta_remove(self, row):
+        """A click on a model's name in the PTA bars: leave it out of every tab (click it again below to bring it back)."""
+        shown = [x["model"] for x in getattr(self, "pta_result", {}).get("models", [])]
+        if not 0 <= row < len(shown):
+            return
+        if len(shown) == 1:
+            self.notify("That's the last model shown.", severity="warning")
+            return
+        model = shown[row]
+        self.app.pta_removed = [m for m in getattr(self.app, "pta_removed", []) if m != model] + [model]
+        self.set_models((self.app.pq_models or set(shown)) - {model})
+
+    def action_pta_restore(self, row):
+        removed = self.pta_removed_now()
+        if 0 <= row < len(removed):
+            self.app.pta_removed.remove(removed[row])
+            self.set_models(self.app.pq_models | {removed[row]})
+
+    def action_pta_restore_all(self):
+        self.app.pta_removed = []
+        self.set_models(set())
+
+    def pta_removed_now(self):
+        """Models removed by a click that are still left out (the Models button may have put them back)."""
+        return [m for m in getattr(self.app, "pta_removed", []) if self.app.pq_models and m not in self.app.pq_models]
+
     # -- insights: charts with what to do next (insights.py); built when the tab is first shown
     @on(TabbedContent.TabActivated)
     def tab_activated(self, event):
@@ -3127,7 +3208,16 @@ class ResultsScreen(Screen):
         # none picked: every model with enough answers (pta.index leaves out the rest, and says so)
         result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos))
         self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
-        self.query_one("#pta-bars", Static).update("\n".join(pta.bars(result)) if result["questions"] else "")
+        self.pta_result = result
+        removed = [m for m in self.pta_removed_now() if m in present]
+        self.query_one("#pta-bars", Static).update(
+            "\n".join(pta.bars(result, click="screen.pta_remove")
+                      + ([f"\n[dim]Click a name to remove it. Removed:[/dim] "
+                          + "  ".join(f"[@click=screen.pta_restore({self.pta_removed_now().index(m)})]{rich_escape(m)}[/]"
+                                      for m in removed)
+                          + "  [dim]·[/dim]  [@click=screen.pta_restore_all]show all[/]"] if removed
+                         else ["\n[dim]Click a name to remove it from view.[/dim]"]))
+            if result["questions"] else "")
         self.query_one("#pta-left-out", Static).update(
             "\n".join(f"[dim]{rich_escape(line)}[/dim]" for line in pta.left_out_lines(result)))
         header, table = pta.table(result)

{tuieval-0.2.0.dev108 → tuieval-0.2.0.dev112}/tests/test_tuieval.py

@@ -1130,6 +1130,70 @@ class AddToRun(unittest.TestCase):
         self.assertEqual(fewer, 4)                     # its 4 questions are no longer counted
         self.assertEqual(out["running"], [False, True])
 
+    def test_tui_copies_reasoning_and_answer(self):
+        path = os.path.join(self.ws, "models.toml")
+        write(path, read(path).replace('"--delay", "0.2"', '"--delay", "1.5"'))   # reasoning first, then a wait
+        fake = os.path.join(self.tmp.name, "bin")
+        os.makedirs(fake)
+        write(os.path.join(fake, "pbcopy"), '#!/bin/sh\ncat > "$CLIP_OUT"\n')
+        os.chmod(os.path.join(fake, "pbcopy"), 0o755)
+        clip = os.path.join(self.tmp.name, "clip.txt")
+        code = textwrap.dedent("""
+            import asyncio, json, os, time
+            from tuieval.tui import AnswerScreen, EvalsApp, RunScreen
+            from textual.widgets import SelectionList
+            def pasted():
+                with open(os.environ["CLIP_OUT"]) as f:
+                    text = f.read()
+                os.remove(os.environ["CLIP_OUT"])
+                return text
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(160, 50)) as pilot:
+                    await pilot.pause()
+                    s = app.screen
+                    s.query_one("#suites", SelectionList).select("apps")
+                    s.selected_models = {"a"}
+                    s.refresh_models()
+                    s.query_one("#tier-certify").value = True
+                    await pilot.pause()
+                    s.action_start()
+                    deadline = time.time() + 180     # CI machines can take long to start servers
+                    while time.time() < deadline:
+                        await pilot.pause(0.1)
+                        if isinstance(app.screen, RunScreen) and app.screen.req_started:
+                            break
+                    run, out = app.screen, {}
+                    await pilot.pause(0.5)            # the reasoning has streamed; the answer waits 1.5 s
+                    await pilot.press("y")
+                    await pilot.pause()
+                    out["live"] = pasted()
+                    deadline = time.time() + 120
+                    while run.running and time.time() < deadline:
+                        await pilot.pause(0.2)
+                    app.push_screen(AnswerScreen(list(run.recent_entries.values())))
+                    await pilot.pause()
+                    for key in "ya":
+                        await pilot.press(key)
+                        await pilot.pause()
+                        out[key] = pasted()
+                    print(json.dumps(out))
+            asyncio.run(go())
+        """)
+        p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=600,
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS="", CLIP_OUT=clip,
+                                    PATH=fake + os.pathsep + os.environ["PATH"]))
+        out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
+        self.assertEqual(out, {"live": "Thinking it over. ", "y": "Thinking it over. ", "a": "ANSWER: 1"}, out)
+
+    def test_copy_falls_back_to_the_terminal(self):
+        from unittest import mock
+        from tuieval.tui import copy_text
+        app = mock.Mock()
+        with mock.patch.dict(os.environ, {"PATH": self.tmp.name}):      # no clipboard tool here
+            self.assertFalse(copy_text(app, "some reasoning"))
+        app.copy_to_clipboard.assert_called_once_with("some reasoning")
+
     def test_tui_counts_ticked_models_and_names_filtered_ones_before_start(self):
         code = textwrap.dedent("""
             import asyncio, json
@@ -1609,6 +1673,23 @@ class PTAIndex(unittest.TestCase):
                         app.screen.fill_pta()                                   # rebuilt: still sorted
                         await pilot.pause(0.2)
                         sorts["kept"] = [str(pta_t.get_row_at(r)[0]) for r in range(pta_t.row_count)]
+                        bars_w = app.screen.query_one("#pta-bars", Static)
+                        shown = [x["model"] for x in app.screen.pta_result["models"]]
+                        await pilot.click(bars_w, offset=(1, 1 + 2 + shown.index("hosted")))   # a click on its name
+                        await pilot.pause(0.3)
+                        clicks = {"removed": [sorted(app.pq_models), [x["model"] for x in app.screen.pta_result["models"]],
+                                              "Removed: hosted" in str(bars_w.render())]}
+                        await pilot.click(bars_w, offset=(1, 1 + 2))                         # the last one stays
+                        await pilot.pause(0.3)
+                        clicks["last"] = sorted(app.pq_models)
+                        app.screen.action_pta_restore(0)
+                        await pilot.pause(0.3)
+                        clicks["restored"] = [sorted(app.pq_models), len(app.screen.pta_result["models"])]
+                        app.screen.action_pta_remove(shown.index("hosted"))
+                        app.screen.action_pta_restore_all()
+                        await pilot.pause(0.3)
+                        clicks["all"] = [sorted(app.pq_models), "Removed" in str(bars_w.render())]
+                        sorts["clicks"] = clicks
                         app.screen.query_one(TabbedContent).active = "tab-failures"
                         await pilot.pause(0.3)
                         f = app.screen.query_one("#failures", DataTable)
@@ -1647,6 +1728,11 @@ class PTAIndex(unittest.TestCase):
             s = got["sorts"]   # a click on a column name sorts by it, a second click reverses
             self.assertEqual((s["up"], s["down"], s["kept"]), (["hosted", "good"], ["good", "hosted"], ["good", "hosted"]))
             self.assertIn("answers right ▼", s["labels"])
+            c = s["clicks"]   # a click on a name in the bars leaves it out of every tab; the last one stays
+            self.assertEqual(c["removed"], [["good"], ["good"], True])
+            self.assertEqual(c["last"], ["good"])
+            self.assertEqual(c["restored"], [["good", "hosted"], 2])
+            self.assertEqual(c["all"], [[], False])
             self.assertEqual((s["tests"][0], s["opened"]), ("apps: d", "d"))
             fails = got["fails"]
             self.assertEqual(fails["all"], [4, "4 failures"])