tuieval 0.2.0.dev94__py3-none-any.whl → 0.2.0.dev96__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev94'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev94')
+__version__ = version = '0.2.0.dev96'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev96')
 
 __commit_id__ = commit_id = None

tuieval/pta.py

@@ -47,7 +47,7 @@ def memory_gb(infos):
 
 
 def index(rows, models=None, memory=None):
-    """{"questions": n shared, "models": [{model, P, T, A, tokens, total_s, passed, answers, alongside,
+    """{"questions": n shared, "models": [{model, P, T, A, cut, cut_pct, tokens, total_s, passed, answers, alongside,
     fewest, fastest, tps, ttft, memory_gb}],
     "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
     most-answered model}, best accuracy first. models: the ones to compare (default: every model in rows
@@ -75,6 +75,9 @@ def index(rows, models=None, memory=None):
                     "total_s": sum(c["secs"] or 0 for c in cells),
                     "passed": sum(c["passed"] for c in cells), "answers": answers,
                     "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
+                    # failed only because max_tokens ran out (overthinking, not wrong yet)
+                    "cut": sum(c["truncated"] for c in cells),
+                    "cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
                     # generation speed and time to first token over the same answers (medians)
                     "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
                     "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
@@ -118,7 +121,8 @@ def bars(result, width=20, markup=True):
     """Lines with a row per model and a bar each for P, T and A: the at-a-glance view. The P and T bars
     are proportional to tokens and time (the longest is the model that used most; shorter is better),
     with how many times the leanest / fastest model's beside them; the A bar is the share of answers
-    right (longer is better, full only when every answer passed). markup=False gives plain text."""
+    right (longer is better, full only when every answer passed), then the share cut off at max_tokens in
+    orange (▓ in plain text) and "✂ n%" beside it. markup=False gives plain text."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
@@ -138,12 +142,18 @@ def bars(result, width=20, markup=True):
                 if v is not None and v < 100:   # full only when every answer passed
                     full = min(full, width - 1)
                 num = "-" if v is None else compare.pct(v / 100)
+                cut = x.get("cut_pct") or 0
+                part = min(width - full, max(1, round(width * cut / 100))) if cut else 0
+                if cut:   # after the accuracy, which stays aligned with the other rows'
+                    num = f"{num:>6}  ✂ {compare.pct(cut / 100)}"
             else:
                 full = _blocks(width, x.get(measure), most[measure])
                 num = times(x.get(shown))
-            bar, rest = "█" * full, "░" * (width - full)
-            bar = f"[{color}]{bar}[/][grey37]{rest}[/]" if markup else bar + rest
-            cells.append(f"{bar} {num:>6}" + " " * (cell_w - width - 7))
+                part = 0
+            bar, cut_bar, rest = "█" * full, "▓" * part, "░" * (width - full - part)
+            bar = (f"[{color}]{bar}[/]" + (f"[#ff8c00]{'█' * part}[/]" if part else "") + f"[grey37]{rest}[/]"
+                   if markup else bar + cut_bar + rest)
+            cells.append(f"{bar} {num:>6}" + " " * max(0, cell_w - width - 1 - max(6, len(num))))
         name = f"{x['model']:<{name_w}}"
         lines.append((f"{rich_escape(name) if markup else name}  " + "".join(cells)).rstrip())
     return lines
@@ -168,10 +178,11 @@ def left_out_lines(result):
 def table(result):
     """(header, rows) for a plain table of the index."""
     n = result["questions"] or 1
-    header = ["model", "tokens/answer", "time/answer", "answers right", "tok/s", "TTFT s", "memory GB"]
+    header = ["model", "tokens/answer", "time/answer", "answers right", "cut off", "tok/s", "TTFT s", "memory GB"]
     rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
              fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
-             f"{x['passed']}/{x['answers']}", _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
+             f"{x['passed']}/{x['answers']}", f"{x.get('cut') or 0}/{x['answers']}",
+             _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
              _num(x.get("memory_gb"), ".1f")] for x in result["models"]]
     return header, rows
 
@@ -204,7 +215,9 @@ def scope_note(result, models):
     return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
             "P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
             "and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
-            "(longer is better).")
+            "(longer is better)"
+            + ("; orange ✂: answers cut off at max_tokens, out of tokens rather than wrong (they count as wrong)."
+               if any(x.get("cut") for x in result["models"]) else "."))
 
 
 def for_engine(e, labels=None, packs=None, results_dir=None):

{tuieval-0.2.0.dev94.dist-info → tuieval-0.2.0.dev96.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev94
+Version: 0.2.0.dev96
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev94.dist-info → tuieval-0.2.0.dev96.dist-info}/RECORD

@@ -1,6 +1,6 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=fdKFKgBcDb2th9hbBPfqkbuerxM6n_GrUhycjixQLz0,535
+tuieval/_version.py,sha256=vq8xfmWFF-z2N4IfHoIiukOF66_VgRgLRvVKdboXuLA,535
 tuieval/builtin.toml,sha256=Nwu7O0qFzcrDAxXO_VZtDPvjVwxZ29OtYOW0BPihZGs,4410
 tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
 tuieval/client.py,sha256=wtACBVsngtWopWWhIc4bB6mEUEi31dvvDePN-PjmXbM,11122
@@ -14,7 +14,7 @@ tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=ZgaZ1m2ePYvDjmddSc3nafnMEn7kTq9WcDpCaJSZleQ,12026
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
-tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
+tuieval/pta.py,sha256=_e6coSXYSzEagWJItyhz-DqIud5ywoVVa7BXbS7kUnY,13183
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=DN6FhC8Zj08N5pwlrCmlkbkTxA2vmT-G5sWJnp-S-AI,50114
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
@@ -48,8 +48,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev94.dist-info/METADATA,sha256=i9PkqtvzVrMcs37nNV4c6RpKMBnQ3skhzmQrqkuUtKU,14580
-tuieval-0.2.0.dev94.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev94.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev94.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev94.dist-info/RECORD,,
+tuieval-0.2.0.dev96.dist-info/METADATA,sha256=tWhbY2NNalpYs0drNts9Vs1vsPHZ-Muy9Nw-34gftRw,14580
+tuieval-0.2.0.dev96.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev96.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev96.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev96.dist-info/RECORD,,