tuieval 0.2.0.dev108__py3-none-any.whl → 0.2.0.dev112__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tuieval/_version.py CHANGED
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.2.0.dev108'
22
- __version_tuple__ = version_tuple = (0, 2, 0, 'dev108')
21
+ __version__ = version = '0.2.0.dev112'
22
+ __version_tuple__ = version_tuple = (0, 2, 0, 'dev112')
23
23
 
24
24
  __commit_id__ = commit_id = None
tuieval/pta.py CHANGED
@@ -117,12 +117,13 @@ def _blocks(width, v, most):
117
117
  return max(1, min(width, round(width * v / most)))
118
118
 
119
119
 
120
- def bars(result, width=20, markup=True):
120
+ def bars(result, width=20, markup=True, click=None):
121
121
  """Lines with a row per model and a bar each for P, T and A: the at-a-glance view. The P and T bars
122
122
  are proportional to tokens and time (the longest is the model that used most; shorter is better),
123
123
  with how many times the leanest / fastest model's beside them; the A bar is the share of answers
124
124
  right (longer is better, full only when every answer passed), then the share cut off at max_tokens in
125
- orange (▓ in plain text) and "(n% truncated)" beside it. markup=False gives plain text."""
125
+ orange (▓ in plain text) and "(n% truncated)" beside it. markup=False gives plain text; click (an
126
+ action, e.g. "screen.remove") makes each model's name a link calling it with the row's index."""
126
127
  if not result["models"]:
127
128
  return []
128
129
  name_w = max(len(x["model"]) for x in result["models"])
@@ -133,7 +134,7 @@ def bars(result, width=20, markup=True):
133
134
  sub = pad + "".join(f"{better:<{cell_w}}" for _, _, better, *_ in LETTERS)
134
135
  lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip(), f"[dim]{sub.rstrip()}[/dim]" if markup else sub.rstrip()]
135
136
  most = {m: max((x.get(m) or 0 for x in result["models"]), default=0) for _, _, _, m, _, _ in LETTERS}
136
- for x in result["models"]:
137
+ for row, x in enumerate(result["models"]):
137
138
  cells = []
138
139
  for k, _, _, measure, shown, color in LETTERS:
139
140
  if k == "A":
@@ -155,7 +156,11 @@ def bars(result, width=20, markup=True):
155
156
  if markup else bar + cut_bar + rest)
156
157
  cells.append(f"{bar} {num:>6}" + " " * max(0, cell_w - width - 1 - max(6, len(num))))
157
158
  name = f"{x['model']:<{name_w}}"
158
- lines.append((f"{rich_escape(name) if markup else name} " + "".join(cells)).rstrip())
159
+ if markup:
160
+ pad_w = name_w - len(x["model"])
161
+ name = (f"[@click={click}({row})]{rich_escape(x['model'])}[/]" + " " * pad_w if click
162
+ else rich_escape(name))
163
+ lines.append((f"{name} " + "".join(cells)).rstrip())
159
164
  return lines
160
165
 
161
166
 
tuieval/tui.py CHANGED
@@ -24,6 +24,8 @@ import glob
24
24
  import json
25
25
  import os
26
26
  import re
27
+ import shutil
28
+ import subprocess
27
29
  import threading
28
30
  import time
29
31
 
@@ -289,6 +291,37 @@ class StreamView(TextArea):
289
291
  self.call_after_refresh(lambda: self.scroll_to(y=max(0, self.max_scroll_y - from_end), animate=False))
290
292
 
291
293
 
294
+ CLIPBOARD_TOOLS = (["pbcopy"], ["wl-copy"], ["xclip", "-selection", "clipboard"], ["xsel", "--clipboard", "--input"],
295
+ ["clip"])
296
+
297
+
298
+ def copy_text(app, text):
299
+ """Put text on the system clipboard with the first tool here that takes it (pbcopy, wl-copy, xclip, xsel,
300
+ clip). Without one, the terminal is asked to (OSC 52), which not every terminal allows. True if a tool did."""
301
+ for cmd in CLIPBOARD_TOOLS:
302
+ if not shutil.which(cmd[0]):
303
+ continue
304
+ try:
305
+ subprocess.run(cmd, input=text.encode("utf-16" if cmd[0] == "clip" else "utf-8"),
306
+ capture_output=True, check=True, timeout=10)
307
+ return True
308
+ except (OSError, subprocess.SubprocessError):
309
+ continue # e.g. xclip without a display: try the next
310
+ app.copy_to_clipboard(text)
311
+ return False
312
+
313
+
314
+ def notify_copied(app, what, text, note=""):
315
+ if not text:
316
+ app.notify(f"No {what} to copy.", severity="warning")
317
+ elif copy_text(app, text):
318
+ app.notify(f"Copied {what}{note}: {len(text):,} chars.")
319
+ else:
320
+ app.notify(f"Sent {what}{note} ({len(text):,} chars) to the terminal's clipboard. If nothing pastes, "
321
+ "the terminal doesn't allow it: install a clipboard tool (wl-copy, xclip or xsel).",
322
+ severity="warning", timeout=10)
323
+
324
+
292
325
  # ------------------------------------------------------------------ answer detail
293
326
  def answer_entry(row):
294
327
  """An answer for AnswerScreen from a compare row (None when the row has no stored answer)."""
@@ -451,7 +484,7 @@ with the model's reasoning and answer as they stream, then recent results with t
451
484
  PTA index parsimony (tokens), time and accuracy per model, a bar each: tokens and time
452
485
  as long as they took (shorter is better; 2.4× = 2.4 times the best), accuracy
453
486
  longer is better. Per-answer numbers, tok/s, TTFT and memory in the table.
454
- Results open here.
487
+ Results open here. Click a model's name to remove it from view (every tab).
455
488
  Production readiness one verdict per model and use case; the table says why and what to run next
456
489
  Per question every question, model by model (filters: pack, level, Models disagree)
457
490
  Failures every wrong answer with the grader's reason (filter: Models); enter opens it
@@ -511,11 +544,14 @@ class AnswerScreen(ModalScreen):
511
544
  Binding("q", "close", "Close", show=False, priority=True),
512
545
  Binding("left_square_bracket", "step(-1)", "Previous", priority=True),
513
546
  Binding("right_square_bracket", "step(1)", "Next", priority=True),
547
+ Binding("y", "copy('reasoning')", "Copy reasoning", priority=True),
548
+ Binding("a", "copy('answer')", "Copy answer", priority=True),
514
549
  ]
515
550
 
516
551
  def __init__(self, entries, index=0, what="answer"):
517
552
  super().__init__()
518
553
  self.entries, self.index, self.what = entries, index, what
554
+ self.texts = {} # what's shown: "reasoning", "answer" -> text, for the copy keys
519
555
 
520
556
  def check_action(self, action, parameters):
521
557
  return len(self.entries) > 1 if action == "step" else True
@@ -550,6 +586,9 @@ class AnswerScreen(ModalScreen):
550
586
  self.index = (self.index + delta) % len(self.entries)
551
587
  self.show()
552
588
 
589
+ def action_copy(self, what):
590
+ notify_copied(self.app, what, self.texts.get(what, ""))
591
+
553
592
  def show(self):
554
593
  e = self.app.engine
555
594
  entry = self.entries[self.index]
@@ -608,6 +647,7 @@ class AnswerScreen(ModalScreen):
608
647
  answer += "\n\n=== TOOL CALLS ===\n" + json.dumps(rec["tool_calls"], indent=2)
609
648
  for wid, text in (("#answer-reasoning", reasoning), ("#answer-answer", answer or "(empty)")):
610
649
  self.query_one(wid, TextArea).load_text(text)
650
+ self.texts = {"reasoning": reasoning, "answer": answer}
611
651
 
612
652
  question, checked = "(test not found)", "(test not found)"
613
653
  if test:
@@ -2163,6 +2203,7 @@ class RunScreen(Screen):
2163
2203
  Binding("c", "cancel_all", "Cancel all", priority=True),
2164
2204
  Binding("f", "toggle_follow", "Pause scroll", priority=True),
2165
2205
  Binding("v", "watch_next", "Next model", priority=True),
2206
+ Binding("y", "copy_reasoning", "Copy reasoning", priority=True),
2166
2207
  Binding("n", "new_run", "More evals", priority=True),
2167
2208
  Binding("r", "results", "Results", priority=True),
2168
2209
  Binding("w", "runs", "Runs", priority=True),
@@ -2182,6 +2223,8 @@ class RunScreen(Screen):
2182
2223
  self.follow = True
2183
2224
  self.buf = {"reasoning": [], "answer": []}
2184
2225
  self.buf_lock = threading.Lock()
2226
+ self.streamed = {"reasoning": [], "answer": []} # the watched answer so far, untrimmed (for y)
2227
+ self.streamed_whole = False # from its start (not joined midway with v)
2185
2228
  self.req_live = {} # model label -> its request now: phase, ~tokens, started, last token
2186
2229
  self.started = time.time()
2187
2230
  self.req_started = None
@@ -2344,6 +2387,7 @@ class RunScreen(Screen):
2344
2387
  v.clear()
2345
2388
  for stream, text in parts.items():
2346
2389
  if text:
2390
+ self.streamed[stream].append(text)
2347
2391
  self.query_one(f"#{stream}", StreamView).append(text, self.follow)
2348
2392
  self.req_tokens += max(1, len(text) // 4)
2349
2393
  if self.req_first is None and self.req_started:
@@ -2364,6 +2408,7 @@ class RunScreen(Screen):
2364
2408
  self.flush_streams()
2365
2409
  self.query_one("#reasoning", StreamView).clear()
2366
2410
  self.query_one("#answer", StreamView).clear()
2411
+ self.streamed, self.streamed_whole = {"reasoning": [], "answer": []}, True
2367
2412
  self.req_started, self.req_tokens, self.req_first = time.time(), 0, None
2368
2413
  job = d["job"]
2369
2414
  rep_note = f" · repeat {d['repeat'] + 1}/{job.repeat}" if job.repeat > 1 else ""
@@ -2731,6 +2776,7 @@ class RunScreen(Screen):
2731
2776
  v.clear()
2732
2777
  self.query_one("#reasoning", StreamView).clear()
2733
2778
  self.query_one("#answer", StreamView).clear()
2779
+ self.streamed, self.streamed_whole = {"reasoning": [], "answer": []}, False
2734
2780
  self.req_started = None
2735
2781
  self.current = f"Streaming [b]{self.watch}[/b] from its next answer…"
2736
2782
  self.refresh_current()
@@ -2748,6 +2794,12 @@ class RunScreen(Screen):
2748
2794
  self.follow = not self.follow
2749
2795
  self.update_overall()
2750
2796
 
2797
+ def action_copy_reasoning(self):
2798
+ """The streaming answer's reasoning so far, all of it (the pane drops the oldest text when it's long)."""
2799
+ self.flush_streams()
2800
+ notify_copied(self.app, "reasoning", "".join(self.streamed["reasoning"]),
2801
+ "" if self.streamed_whole else " (from midway: streaming joined after it began)")
2802
+
2751
2803
  @on(DataTable.RowSelected, "#recent")
2752
2804
  def recent_selected(self, event):
2753
2805
  keys = [k for k in event.data_table.rows if k in self.recent_entries]
@@ -3032,22 +3084,51 @@ class ResultsScreen(Screen):
3032
3084
  models = sorted(counts, key=lambda m: (-counts[m], m))
3033
3085
 
3034
3086
  def picked(chosen):
3035
- if chosen is None:
3036
- return
3037
- first = chosen and not self.app.pq_models
3038
- self.app.pq_models = chosen # kept for this session, so reopening Results keeps it
3039
- if first: # a fair comparison needs the same questions: turn it on (you can untick it)
3040
- with self.query_one("#pq-shared", Checkbox).prevent(Checkbox.Changed):
3041
- self.query_one("#pq-shared", Checkbox).value = True
3042
- # after the picker has closed: a table rebuilt under a modal keeps its header-only widths
3043
- self.call_after_refresh(self.fill_per_question)
3044
- self.call_after_refresh(self.fill_pta)
3045
- self.call_after_refresh(self.fill_failures)
3046
- self.insights_stale = True
3047
- if self.query_one(TabbedContent).active == "tab-insights":
3048
- self.call_after_refresh(self.fill_insights)
3087
+ if chosen is not None:
3088
+ self.set_models(chosen)
3049
3089
  self.app.push_screen(ModelPickScreen(models, counts, self.app.pq_models), picked)
3050
3090
 
3091
+ def set_models(self, chosen):
3092
+ """The models every Results tab compares (none: all of them)."""
3093
+ first = chosen and not self.app.pq_models
3094
+ self.app.pq_models = chosen # kept for this session, so reopening Results keeps it
3095
+ if first: # a fair comparison needs the same questions: turn it on (you can untick it)
3096
+ with self.query_one("#pq-shared", Checkbox).prevent(Checkbox.Changed):
3097
+ self.query_one("#pq-shared", Checkbox).value = True
3098
+ # after a picker has closed: a table rebuilt under a modal keeps its header-only widths
3099
+ self.call_after_refresh(self.fill_per_question)
3100
+ self.call_after_refresh(self.fill_pta)
3101
+ self.call_after_refresh(self.fill_failures)
3102
+ self.insights_stale = True
3103
+ if self.query_one(TabbedContent).active == "tab-insights":
3104
+ self.call_after_refresh(self.fill_insights)
3105
+
3106
+ def action_pta_remove(self, row):
3107
+ """A click on a model's name in the PTA bars: leave it out of every tab (click it again below to bring it back)."""
3108
+ shown = [x["model"] for x in getattr(self, "pta_result", {}).get("models", [])]
3109
+ if not 0 <= row < len(shown):
3110
+ return
3111
+ if len(shown) == 1:
3112
+ self.notify("That's the last model shown.", severity="warning")
3113
+ return
3114
+ model = shown[row]
3115
+ self.app.pta_removed = [m for m in getattr(self.app, "pta_removed", []) if m != model] + [model]
3116
+ self.set_models((self.app.pq_models or set(shown)) - {model})
3117
+
3118
+ def action_pta_restore(self, row):
3119
+ removed = self.pta_removed_now()
3120
+ if 0 <= row < len(removed):
3121
+ self.app.pta_removed.remove(removed[row])
3122
+ self.set_models(self.app.pq_models | {removed[row]})
3123
+
3124
+ def action_pta_restore_all(self):
3125
+ self.app.pta_removed = []
3126
+ self.set_models(set())
3127
+
3128
+ def pta_removed_now(self):
3129
+ """Models removed by a click that are still left out (the Models button may have put them back)."""
3130
+ return [m for m in getattr(self.app, "pta_removed", []) if self.app.pq_models and m not in self.app.pq_models]
3131
+
3051
3132
  # -- insights: charts with what to do next (insights.py); built when the tab is first shown
3052
3133
  @on(TabbedContent.TabActivated)
3053
3134
  def tab_activated(self, event):
@@ -3127,7 +3208,16 @@ class ResultsScreen(Screen):
3127
3208
  # none picked: every model with enough answers (pta.index leaves out the rest, and says so)
3128
3209
  result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos))
3129
3210
  self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
3130
- self.query_one("#pta-bars", Static).update("\n".join(pta.bars(result)) if result["questions"] else "")
3211
+ self.pta_result = result
3212
+ removed = [m for m in self.pta_removed_now() if m in present]
3213
+ self.query_one("#pta-bars", Static).update(
3214
+ "\n".join(pta.bars(result, click="screen.pta_remove")
3215
+ + ([f"\n[dim]Click a name to remove it. Removed:[/dim] "
3216
+ + " ".join(f"[@click=screen.pta_restore({self.pta_removed_now().index(m)})]{rich_escape(m)}[/]"
3217
+ for m in removed)
3218
+ + " [dim]·[/dim] [@click=screen.pta_restore_all]show all[/]"] if removed
3219
+ else ["\n[dim]Click a name to remove it from view.[/dim]"]))
3220
+ if result["questions"] else "")
3131
3221
  self.query_one("#pta-left-out", Static).update(
3132
3222
  "\n".join(f"[dim]{rich_escape(line)}[/dim]" for line in pta.left_out_lines(result)))
3133
3223
  header, table = pta.table(result)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: tuieval
3
- Version: 0.2.0.dev108
3
+ Version: 0.2.0.dev112
4
4
  Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
5
5
  Project-URL: Homepage, https://github.com/ashe-wb/tuieval
6
6
  Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -66,7 +66,7 @@ While a run goes, you watch each question's reasoning and answer stream in:
66
66
 
67
67
  ![tuieval's run screen with a model's reasoning streaming in for a coding question](https://raw.githubusercontent.com/ashe-wb/tuieval/main/docs/images/tui-run-reasoning.png)
68
68
 
69
- Afterwards, every answer keeps its reasoning. Enter on a result opens it with the grade and each check, the full reasoning and the answer; `[` and `]` step through the others:
69
+ Afterwards, every answer keeps its reasoning. Enter on a result opens it with the grade and each check, the full reasoning and the answer; `[` and `]` step through the others, `y` copies the reasoning and `a` the answer (on the run screen, `y` copies the reasoning streamed so far):
70
70
 
71
71
  ![one answer in full: the grade and its checks, the model's reasoning and its code](https://raw.githubusercontent.com/ashe-wb/tuieval/main/docs/images/tui-answer-reasoning.png)
72
72
 
@@ -1,6 +1,6 @@
1
1
  tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
2
2
  tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
3
- tuieval/_version.py,sha256=CdiAJD4jTvwX1BMRCxNdagh-cHCpvX9EPh7QkYx4nYY,537
3
+ tuieval/_version.py,sha256=W7ovaJXOHI4LeAQAjKdSNMOHKKyCHZRZ3K5cv33MJLk,537
4
4
  tuieval/builtin.toml,sha256=Nwu7O0qFzcrDAxXO_VZtDPvjVwxZ29OtYOW0BPihZGs,4410
5
5
  tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
6
6
  tuieval/client.py,sha256=wtACBVsngtWopWWhIc4bB6mEUEi31dvvDePN-PjmXbM,11122
@@ -14,12 +14,12 @@ tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
14
14
  tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
15
15
  tuieval/packs.py,sha256=ZgaZ1m2ePYvDjmddSc3nafnMEn7kTq9WcDpCaJSZleQ,12026
16
16
  tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
17
- tuieval/pta.py,sha256=lvcPZ-taIEu2RHo-IiWb31OYtmkoCrsdulrF4-T7Csk,13243
17
+ tuieval/pta.py,sha256=nU0HLl28CoRlmU4Hw3rqy6i03e7bG0Eto_gShS_eqUM,13552
18
18
  tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
19
19
  tuieval/run_evals.py,sha256=DN6FhC8Zj08N5pwlrCmlkbkTxA2vmT-G5sWJnp-S-AI,50114
20
20
  tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
21
21
  tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
22
- tuieval/tui.py,sha256=itrelvuREKLy8W6QNI1moeMNBLFoUyuracYJyj05sx8,190527
22
+ tuieval/tui.py,sha256=UFVL89F5AkoN7KUMoDfFI7cCZChMpoutwceJtz_4IrE,195167
23
23
  tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
24
24
  tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
25
25
  tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -48,8 +48,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
48
48
  tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
49
49
  tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
50
50
  tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
51
- tuieval-0.2.0.dev108.dist-info/METADATA,sha256=XVqm_fJia2U5_yH3ZLbg-CfyxJL4D1eIWy54qGoRX54,14691
52
- tuieval-0.2.0.dev108.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
53
- tuieval-0.2.0.dev108.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
54
- tuieval-0.2.0.dev108.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
55
- tuieval-0.2.0.dev108.dist-info/RECORD,,
51
+ tuieval-0.2.0.dev112.dist-info/METADATA,sha256=toA2Ug8fgr_Fd--c7Qa4Lp9erPs35Ur-Cl1baVSDP2E,14798
52
+ tuieval-0.2.0.dev112.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
53
+ tuieval-0.2.0.dev112.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
54
+ tuieval-0.2.0.dev112.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
55
+ tuieval-0.2.0.dev112.dist-info/RECORD,,