tuieval 0.2.0.dev94__py3-none-any.whl → 0.2.0.dev98__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev94'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev94')
+__version__ = version = '0.2.0.dev98'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev98')
 
 __commit_id__ = commit_id = None

tuieval/pta.py

@@ -47,7 +47,7 @@ def memory_gb(infos):
 
 
 def index(rows, models=None, memory=None):
-    """{"questions": n shared, "models": [{model, P, T, A, tokens, total_s, passed, answers, alongside,
+    """{"questions": n shared, "models": [{model, P, T, A, cut, cut_pct, tokens, total_s, passed, answers, alongside,
     fewest, fastest, tps, ttft, memory_gb}],
     "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
     most-answered model}, best accuracy first. models: the ones to compare (default: every model in rows
@@ -75,6 +75,9 @@ def index(rows, models=None, memory=None):
                     "total_s": sum(c["secs"] or 0 for c in cells),
                     "passed": sum(c["passed"] for c in cells), "answers": answers,
                     "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
+                    # failed only because max_tokens ran out (overthinking, not wrong yet)
+                    "cut": sum(c["truncated"] for c in cells),
+                    "cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
                     # generation speed and time to first token over the same answers (medians)
                     "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
                     "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
@@ -118,7 +121,8 @@ def bars(result, width=20, markup=True):
     """Lines with a row per model and a bar each for P, T and A: the at-a-glance view. The P and T bars
     are proportional to tokens and time (the longest is the model that used most; shorter is better),
     with how many times the leanest / fastest model's beside them; the A bar is the share of answers
-    right (longer is better, full only when every answer passed). markup=False gives plain text."""
+    right (longer is better, full only when every answer passed), then the share cut off at max_tokens in
+    orange (▓ in plain text) and "✂ n%" beside it. markup=False gives plain text."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
@@ -138,12 +142,18 @@ def bars(result, width=20, markup=True):
                 if v is not None and v < 100:   # full only when every answer passed
                     full = min(full, width - 1)
                 num = "-" if v is None else compare.pct(v / 100)
+                cut = x.get("cut_pct") or 0
+                part = min(width - full, max(1, round(width * cut / 100))) if cut else 0
+                if cut:   # after the accuracy, which stays aligned with the other rows'
+                    num = f"{num:>6}  ✂ {compare.pct(cut / 100)}"
             else:
                 full = _blocks(width, x.get(measure), most[measure])
                 num = times(x.get(shown))
-            bar, rest = "█" * full, "░" * (width - full)
-            bar = f"[{color}]{bar}[/][grey37]{rest}[/]" if markup else bar + rest
-            cells.append(f"{bar} {num:>6}" + " " * (cell_w - width - 7))
+                part = 0
+            bar, cut_bar, rest = "█" * full, "▓" * part, "░" * (width - full - part)
+            bar = (f"[{color}]{bar}[/]" + (f"[#ff8c00]{'█' * part}[/]" if part else "") + f"[grey37]{rest}[/]"
+                   if markup else bar + cut_bar + rest)
+            cells.append(f"{bar} {num:>6}" + " " * max(0, cell_w - width - 1 - max(6, len(num))))
         name = f"{x['model']:<{name_w}}"
         lines.append((f"{rich_escape(name) if markup else name}  " + "".join(cells)).rstrip())
     return lines
@@ -168,10 +178,11 @@ def left_out_lines(result):
 def table(result):
     """(header, rows) for a plain table of the index."""
     n = result["questions"] or 1
-    header = ["model", "tokens/answer", "time/answer", "answers right", "tok/s", "TTFT s", "memory GB"]
+    header = ["model", "tokens/answer", "time/answer", "answers right", "cut off", "tok/s", "TTFT s", "memory GB"]
     rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
              fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
-             f"{x['passed']}/{x['answers']}", _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
+             f"{x['passed']}/{x['answers']}", f"{x.get('cut') or 0}/{x['answers']}",
+             _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
              _num(x.get("memory_gb"), ".1f")] for x in result["models"]]
     return header, rows
 
@@ -204,7 +215,9 @@ def scope_note(result, models):
     return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
             "P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
             "and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
-            "(longer is better).")
+            "(longer is better)"
+            + ("; orange ✂: answers cut off at max_tokens, out of tokens rather than wrong (they count as wrong)."
+               if any(x.get("cut") for x in result["models"]) else "."))
 
 
 def for_engine(e, labels=None, packs=None, results_dir=None):

tuieval/tui.py

@@ -355,6 +355,38 @@ def rich_escape(text):
     return text.replace("[", r"\[")
 
 
+def mid_trim(text, width, head=None):
+    """text cut to width cells with … in the middle, so both ends of a model name show (its family and
+    its server suffix, e.g. -llama / -vllm). head: how many leading characters to keep (default: half)."""
+    if cell_len(text) <= width or width < 5:
+        return text
+    head = width - 1 - width // 2 if head is None else min(head, width - 2)
+    return text[:head] + "…" + text[head - width + 1:]
+
+
+def trim_names(names, width):
+    """names cut to width (mid_trim), still telling them apart: when cutting the middle would make two
+    alike, the cut moves into the start they all share; if that's not enough either, names stay whole."""
+    trimmed = [mid_trim(n, width) for n in names]
+    if len(set(trimmed)) == len(set(names)):
+        return trimmed
+    shared = len(os.path.commonprefix(list(names)))
+    head = width - 1 - max(len(n) - shared for n in names)   # keep every name's end from where they differ
+    if head < 1:
+        return list(names)
+    trimmed = [mid_trim(n, width, head=head) for n in names]
+    return trimmed if len(set(trimmed)) == len(set(names)) else list(names)
+
+
+def fit_first_column(header, rows, room, floor=24):
+    """rows with the first cell trimmed in the middle (mid_trim) so a table of these columns fits room
+    cells across, but never below floor cells. Cells are Rich markup; each column adds 2 cells of padding."""
+    rest = sum(2 + max(cell_len(h), *(Text.from_markup(r[i]).cell_len for r in rows))
+               for i, h in enumerate(header) if i)
+    names = trim_names([r[0] for r in rows], max(floor, room - rest - 2))
+    return [[n] + r[1:] for n, r in zip(names, rows)]
+
+
 class BlockDumper(yaml.SafeDumper):
     """Multi-line strings (reference code, expected text) as readable | blocks, not escaped \\n."""
 
@@ -2981,6 +3013,9 @@ class ResultsScreen(Screen):
         self.query_one("#pta-left-out", Static).update(
             "\n".join(f"[dim]{rich_escape(line)}[/dim]" for line in pta.left_out_lines(result)))
         header, table = pta.table(result)
+        self.pta_shown = bool(table)    # to refit on resize
+        if table:
+            table = fit_first_column(header, table, (self.size.width or self.app.size.width) - 3)
         for i, h in enumerate(header):   # sized to the contents: added before the rows, a column keeps its header's width
             t.add_column(h, width=max(len(h), *(len(r[i]) for r in table)) if table else None)
         t.add_rows(table)
@@ -3065,6 +3100,14 @@ class ResultsScreen(Screen):
                 f"{os.path.basename(os.path.dirname(p))} · {os.path.basename(p)[:-5]}" for p in sorted(self.superseded))
                 + ". Only the new run's answers (◐) are shown for these.[/yellow]" if self.superseded else ""))
 
+    def on_resize(self, event):
+        """Refit the tables whose model names are trimmed to the window's width."""
+        if getattr(self, "pq_summary_args", None):
+            self.fill_pq_summary(*self.pq_summary_args)
+            self.sort_table(self.query_one("#pq-summary", DataTable))
+        if getattr(self, "pta_shown", False):
+            self.fill_pta()
+
     def fill_pq_summary(self, tests, models):
         """One row per shown model over the questions they ALL answered, so totals compare fairly;
         with two models, a head-to-head line."""
@@ -3081,9 +3124,11 @@ class ResultsScreen(Screen):
             title.update("[yellow]These models have no question in common (with the filters above).[/yellow]")
             table.display = False
             return
+        self.pq_summary_args = (tests, models)    # to refit on resize
         rows = compare.compare_models(shared, models)
-        table.add_columns("Model", "Total time", "Passed", "Median tokens", "Total tokens", "Tokens per ✓", "Median s",
-                          "Avg prompt tok/s", "Avg gen tok/s", "Fewest tokens", "Fastest", "Cut off")
+        # short headings so every column fits; the line above the table says what each one is
+        header = ("Model", "Time", "Passed", "Med tok", "Tokens", "Tok/✓", "Med s", "Prompt t/s", "Gen t/s",
+                  "Fewest", "Fastest", "Cut off")
         best = {k: max((r[k] for r in rows if r[k] is not None), default=None)
                 for k in ("passed", "fewest", "fastest", "avg_pp_tps", "avg_gen_tps")}
         low = {k: min(r[k] for r in rows if r[k] is not None) if any(r[k] is not None for r in rows) else None
@@ -3092,9 +3137,10 @@ class ResultsScreen(Screen):
         def hi(v, text, key, good):
             ref = best.get(key) if good == "max" else low.get(key)
             return f"[bold green]{text}[/bold green]" if v is not None and v == ref and len(rows) > 1 else text
+        cells = []
         for r in rows:
             tt = r["total_secs"]
-            table.add_row(r["model"],
+            cells.append([r["model"],
                           hi(tt, ("-" if not tt else f"{tt:.1f}s" if tt < 60 else fmt_secs(tt)), "total_secs", "min"),
                           hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max"),
                           hi(r["med_tokens"], f"{r['med_tokens']:,.0f}" if r["med_tokens"] else "-", "med_tokens", "min"),
@@ -3107,7 +3153,9 @@ class ResultsScreen(Screen):
                              "max"),
                           hi(r["fewest"], str(r["fewest"]), "fewest", "max"),
                           hi(r["fastest"], str(r["fastest"]), "fastest", "max"),
-                          f"[red]{r['cut']}[/red]" if r["cut"] else "0")
+                          f"[red]{r['cut']}[/red]" if r["cut"] else "0"])
+        table.add_columns(*header)
+        table.add_rows(fit_first_column(header, cells, (self.size.width or self.app.size.width) - 3))
         line = f"[b]Comparison[/b] over the {len(shared)} questions all {len(models)} answered"
         if len(models) == 2:
             h = compare.head_to_head(shared, *models)
@@ -3121,9 +3169,10 @@ class ResultsScreen(Screen):
         if shared_speed:
             line += (f"\n[yellow]Some answers of {', '.join(shared_speed)} ran alongside other models, which "
                      "shared the machine's speed: compare their times with care.[/yellow]")
-        title.update(line + "  [dim]Fewest tokens / Fastest = questions won; ties count for both. Total tokens / "
-                     "Total time add up each question's median. Avg tok/s = all tokens over all the time they took "
-                     "(cached prompts left out).[/dim]")
+        title.update(line + "  [dim]Time / Tokens: each question's median, added up. Med tok / Med s: median "
+                     "per answer. Tok/✓: tokens per answer right. Prompt t/s / Gen t/s: all tokens over all the time "
+                     "they took (cached prompts left out). Fewest / Fastest: questions won; ties count for both. "
+                     "Cut off: answers that hit max_tokens.[/dim]")
 
     @staticmethod
     def pq_cell(c, best, here):
@@ -3277,7 +3326,7 @@ class EvalsApp(App):
     #pq-controls Select { width: 22; margin-right: 1; }
     #pq-models { margin-right: 1; min-width: 20; }
     #pq-compare-title { padding: 0 1; height: auto; }
-    #pq-summary { height: auto; max-height: 10; }
+    #pq-summary { height: auto; }   /* every picked model, no scrolling: the tab scrolls instead */
     #pq-legend { padding: 0 1; height: auto; }
     /* Tabs must read as tabs, not as a line of text: each tab is a button-like chip on its own band,
        the active one filled with the accent colour. (Textual's default: dimmed words on the background.) */

{tuieval-0.2.0.dev94.dist-info → tuieval-0.2.0.dev98.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev94
+Version: 0.2.0.dev98
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev94.dist-info → tuieval-0.2.0.dev98.dist-info}/RECORD

@@ -1,6 +1,6 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=fdKFKgBcDb2th9hbBPfqkbuerxM6n_GrUhycjixQLz0,535
+tuieval/_version.py,sha256=NuUT-snuivwhKl0HVYJzwsMjoUUs7AmKaU0peNajn5w,535
 tuieval/builtin.toml,sha256=Nwu7O0qFzcrDAxXO_VZtDPvjVwxZ29OtYOW0BPihZGs,4410
 tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
 tuieval/client.py,sha256=wtACBVsngtWopWWhIc4bB6mEUEi31dvvDePN-PjmXbM,11122
@@ -14,12 +14,12 @@ tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=ZgaZ1m2ePYvDjmddSc3nafnMEn7kTq9WcDpCaJSZleQ,12026
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
-tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
+tuieval/pta.py,sha256=_e6coSXYSzEagWJItyhz-DqIud5ywoVVa7BXbS7kUnY,13183
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=DN6FhC8Zj08N5pwlrCmlkbkTxA2vmT-G5sWJnp-S-AI,50114
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=1JpAtIpq4zJAmiPyg5NYlQQm8lYuhoP1DiRM4-exHsw,181966
+tuieval/tui.py,sha256=yO69K_LfK5pfUZkQgxA7UqFbXK7BtDr6Y6A8JZCBA9E,184628
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -48,8 +48,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev94.dist-info/METADATA,sha256=i9PkqtvzVrMcs37nNV4c6RpKMBnQ3skhzmQrqkuUtKU,14580
-tuieval-0.2.0.dev94.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev94.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev94.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev94.dist-info/RECORD,,
+tuieval-0.2.0.dev98.dist-info/METADATA,sha256=Ek1lrDyIzB45-pcS0G-a06E_bi0drguKYdd-JH4gR3M,14580
+tuieval-0.2.0.dev98.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev98.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev98.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev98.dist-info/RECORD,,