tuieval 0.2.0.dev65__py3-none-any.whl → 0.2.0.dev66__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev65'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev65')
+__version__ = version = '0.2.0.dev66'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev66')
 
 __commit_id__ = commit_id = None

tuieval/pta.py

@@ -1,4 +1,4 @@
-"""PTA index: parsimony (tokens), time and accuracy per model, a bar each (longer = better).
+"""PTA index: parsimony (tokens), time and accuracy per model, a bar each.
 
     P  parsimony (tokens)  the tokens (reasoning and answer) used on every question compared (each
                            question at its median over repeats), shown as how many times the leanest
@@ -7,8 +7,10 @@
                            model's ("28×").
     A  accuracy            the share of answers to those questions that passed (%).
 
-Behind the P and T bars is a 0-100 score: 100 for the best, POINTS_PER_DOUBLING points less for every
-doubling (0 at about 100 times more), so models far behind the best still get a visible bar.
+The P and T bars are as long as the tokens and time used (the model that used the most fills its bar,
+the others in proportion, at least one block), so shorter is better; the A bar: longer is better.
+P and T also carry a 0-100 score (100 for the best, POINTS_PER_DOUBLING points less for every
+doubling), used to order models with the same accuracy.
 
 Models are compared on the questions all of them answered, so one that skipped some never looks
 faster. Server and connection errors aren't answers. By default a model with fewer than half as many
@@ -93,9 +95,11 @@ def index(rows, models=None, memory=None):
             "most": most}
 
 
-# (letter, name, what the number beside the bar shows, colour); the bar itself is the 0-100 score
-LETTERS = (("P", "parsimony (tokens)", "tokens_x", "cyan"), ("T", "time", "time_x", "magenta"),
-           ("A", "accuracy", "A", "green"))
+# (letter, name, which way is better, what the bar measures, the number beside it, colour). P and T bars
+# are the amount itself (tokens, time: shorter is better); the A bar is the share right (longer is better).
+LETTERS = (("P", "parsimony (tokens)", "▼ shorter is better", "tokens", "tokens_x", "cyan"),
+           ("T", "time", "▼ shorter is better", "total_s", "time_x", "magenta"),
+           ("A", "accuracy", "▲ longer is better", "A", "A", "green"))
 
 
 def times(v):
@@ -103,26 +107,41 @@ def times(v):
     return "-" if v is None else f"{v:.1f}×" if v < 9.95 else f"{v:.0f}×"
 
 
+def _blocks(width, v, most):
+    """How many of width blocks a bar fills: v as a share of most, at least one block when v > 0."""
+    if not v or not most:
+        return 0
+    return max(1, min(width, round(width * v / most)))
+
+
 def bars(result, width=20, markup=True):
-    """Lines with a row per model and a bar per score (P, T, A), longer = better: the at-a-glance view.
-    Beside P and T, how many times the leanest / fastest model's tokens / time; beside A, the percentage.
-    markup=False gives plain text (the report)."""
+    """Lines with a row per model and a bar each for P, T and A: the at-a-glance view. The P and T bars
+    are proportional to tokens and time (the longest is the model that used most; shorter is better),
+    with how many times the leanest / fastest model's beside them; the A bar is the share of answers
+    right (longer is better, full only when every answer passed). markup=False gives plain text."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
-    # each column: the bar, a space, "100%" or "9.8×" (5), then a gap; at least as wide as its heading
-    cell_w = max(width + 9, *(len(f"{k} {name}") + 2 for k, name, _, _ in LETTERS))
-    head = " " * (name_w + 2) + "".join(f"{f'{k} {name}':<{cell_w}}" for k, name, _, _ in LETTERS)
-    lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip()]
+    # each column: the bar, a space, "100%" or "9.8×" (6), then a gap; at least as wide as its heading
+    cell_w = max(width + 9, *(len(t) + 2 for k, name, better, *_ in LETTERS for t in (f"{k} {name}", better)))
+    pad = " " * (name_w + 2)
+    head = pad + "".join(f"{f'{k} {name}':<{cell_w}}" for k, name, *_ in LETTERS)
+    sub = pad + "".join(f"{better:<{cell_w}}" for _, _, better, *_ in LETTERS)
+    lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip(), f"[dim]{sub.rstrip()}[/dim]" if markup else sub.rstrip()]
+    most = {m: max((x.get(m) or 0 for x in result["models"]), default=0) for _, _, _, m, _, _ in LETTERS}
     for x in result["models"]:
         cells = []
-        for k, _, shown, color in LETTERS:
-            v = x[k]
-            full = 0 if v is None else round(width * max(0, min(100, v)) / 100)
-            if v is not None and v < 100:   # full only when perfect (accuracy) or the best (P, T)
-                full = min(full, width - 1)
+        for k, _, _, measure, shown, color in LETTERS:
+            if k == "A":
+                v = x["A"]
+                full = 0 if v is None else round(width * max(0, min(100, v)) / 100)
+                if v is not None and v < 100:   # full only when every answer passed
+                    full = min(full, width - 1)
+                num = "-" if v is None else compare.pct(v / 100)
+            else:
+                full = _blocks(width, x.get(measure), most[measure])
+                num = times(x.get(shown))
             bar, rest = "█" * full, "░" * (width - full)
-            num = (("-" if v is None else compare.pct(v / 100)) if shown == "A" else times(x.get(shown)))
             bar = f"[{color}]{bar}[/][grey37]{rest}[/]" if markup else bar + rest
             cells.append(f"{bar} {num:>6}" + " " * (cell_w - width - 7))
         name = f"{x['model']:<{name_w}}"
@@ -183,8 +202,9 @@ def scope_note(result, models):
     if not result["questions"]:
         return "These models have no question in common yet, so there's nothing to compare."
     return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
-            "Longer bars are better. P parsimony (tokens) and T time: how many times the leanest / fastest "
-            "model's (1.0×, ★ in the table). A accuracy: answers right.")
+            "P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
+            "and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
+            "(longer is better).")
 
 
 def for_engine(e, labels=None, packs=None, results_dir=None):

tuieval/tui.py

@@ -332,9 +332,10 @@ with the model's reasoning and answer as they stream, then recent results with t
   v  with several models at a time: stream the next one (k skips the one streaming)""",
     "ResultsScreen": """[b]Results[/b]
 
-  PTA index              parsimony (tokens), time and accuracy per model, a bar each (longer = better;
-                         2.4× = 2.4 times the best), with per-answer numbers, tok/s, TTFT and memory in
-                         the table. Results open here.
+  PTA index              parsimony (tokens), time and accuracy per model, a bar each: tokens and time
+                         as long as they took (shorter is better; 2.4× = 2.4 times the best), accuracy
+                         longer is better. Per-answer numbers, tok/s, TTFT and memory in the table.
+                         Results open here.
   Production readiness   one verdict per model and use case; the table says why and what to run next
   Per question           every question, model by model (filters: pack, level, Models disagree)
   Failures               every wrong answer with the grader's reason; enter opens it in full

{tuieval-0.2.0.dev65.dist-info → tuieval-0.2.0.dev66.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev65
+Version: 0.2.0.dev66
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev65.dist-info → tuieval-0.2.0.dev66.dist-info}/RECORD

@@ -1,6 +1,6 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=QkealilRB02_mYePp9PrQfXW0yGA1qHLQzzPeETGVN4,535
+tuieval/_version.py,sha256=3oGXG-ulb5H7C2aLJwOF-Gqqfw6csmjHt4syzml7C4M,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=fn6z0cupM-aW3fY5fFgnS-HmOQmMzlDg-U76Y6LgL3o,4510
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
@@ -12,12 +12,12 @@ tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
-tuieval/pta.py,sha256=6HwPMvTTJFoZjO-2ZurucJrp1itIzpDTKL1xvYP9W0w,10888
+tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=1zNkCdFt744cqpebqiyj-sZrbOI22c6IYBcXk9koz20,46538
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=lrIv4z21s9OG809706Z-lE2-TZ6gg8z0AjaJ_zW7zkw,161021
+tuieval/tui.py,sha256=LPnUYbynZiHUBv_d7TNttd7smoTlphPHEa5yoGCvU6s,161108
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=VB-0LTxR9y8bJHT9RicxtE9K7cUHBdCu40hOCrsbmA4,19150
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -46,8 +46,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev65.dist-info/METADATA,sha256=Wd-olyVQVqZyQTb817isEBh7GUdLGbBQDxy2Ioc7iBw,13996
-tuieval-0.2.0.dev65.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev65.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev65.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev65.dist-info/RECORD,,
+tuieval-0.2.0.dev66.dist-info/METADATA,sha256=3R7yYobJYnLLsMP_OHfy74PYErGtGBOf50xMtXpZSXA,13996
+tuieval-0.2.0.dev66.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev66.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev66.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev66.dist-info/RECORD,,