tuieval 0.2.0.dev141__tar.gz → 0.2.0.dev143__tar.gz

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- PTA index: models scoring below 85% are left out by default (listed with their score), unless every model is. The **Score ≥** dropdown (`tuieval pta --min-score N`) sets the minimum, Any score shows all; picked models are always shown.
 - Per question → Comparison: a **Pass rate** column beside Passed, the share of answers right with repeats included (shown when some model has repeats).
 - Repeats (Setup, `tuieval run --repeat N`) add N rounds to the saved answers: ×1 run with 2 gives ×3, not ×2. Retest, a blank count and presets still mean the total.
 - Answers stuck in a loop (one short unit repeated over 2000 chars, in the reasoning or the answer) are stopped as they stream and failed as `LOOP`, instead of running to max_tokens. `[defaults] loop_stop_chars` sets the length; 0 turns it off.

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev141
+Version: 0.2.0.dev143
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/docs/models.md

@@ -111,7 +111,7 @@ The **PTA index** puts parsimony (tokens), time and accuracy side by side, as a
 
 The P and T bars are as long as the tokens and time used: the model that used the most fills its bar, the others in proportion (at least one block), so **shorter is better**. The A bar is the share of answers right, so **longer is better**; it's full only when every answer passed. Each heading says which way is better.
 
-Models are compared on the questions all of them answered, so one that skipped some never looks faster. Server and connection errors aren't answers: a model with only those is listed as having none. A model with fewer than half as many answers as the most-answered one is left out (and listed), or it would shrink the shared questions for everyone. Pick the models and pack to compare in the TUI, or use `--only` and `--packs`: models you pick are always compared.
+Models are compared on the questions all of them answered, so one that skipped some never looks faster. Server and connection errors aren't answers: a model with only those is listed as having none. A model with fewer than half as many answers as the most-answered one is left out (and listed), or it would shrink the shared questions for everyone. So is one scoring below 85% (listed with its score), unless every model does: the rest are then compared again, on the questions they all answered. The **Score ≥** dropdown in the tab (or `tuieval pta --min-score N`) sets that minimum; **Any score** (`0`) shows every model. Pick the models and pack to compare in the TUI, or use `--only` and `--packs`: models you pick are always compared.
 
 ## Insights
 

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev141'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev141')
+__version__ = version = '0.2.0.dev143'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev143')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/src/tuieval/pta.py

@@ -21,7 +21,8 @@ it varies). A ×1 model's accuracy rests on one answer per question, so it is ro
 Models are compared on the questions all of them answered, so one that skipped some never looks
 faster. Server and connection errors aren't answers. By default a model with fewer than half as many
 answers as the most-answered model is left out (listed, so it can be picked), or it would shrink the
-shared questions for everyone. There's no combined number: the three stay side by side, and verdicts
+shared questions for everyone, and so is one that scores below MIN_SCORE (also listed), unless every
+model does. There's no combined number: the three stay side by side, and verdicts
 and critical failures stay where they are (verdict.py).
 """
 import math
@@ -31,6 +32,8 @@ from rich.markup import escape as rich_escape
 from . import compare
 
 MIN_SHARE = 0.5   # fewer answers than this share of the most-answered model's: left out by default
+MIN_SCORE = 85    # a score (partial credit, %) below this: left out by default, unless every model's is
+MIN_SCORES = (0, 50, 70, 80, 85, 90, 95)   # the choices offered (0: every score)
 POINTS_PER_DOUBLING = 15
 
 
@@ -55,13 +58,14 @@ def memory_gb(infos):
 SORTS = ("accuracy", "score")
 
 
-def index(rows, models=None, memory=None, sort="accuracy"):
+def index(rows, models=None, memory=None, sort="accuracy", min_score=MIN_SCORE):
     """{"questions": n shared, "models": [{model, P, T, A, S, cut, cut_pct, tokens, total_s, passed, answers, alongside,
     fewest, fastest, tps, ttft, memory_gb, repeats: (fewest, most) answers per question}],
-    "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
-    most-answered model}, best accuracy (sort="accuracy") or score (sort="score") first. S: the mean
+    "left_out": [(model, questions answered)], "low_score": [(model, score %) left out below MIN_SCORE],
+    "no_answers": [model], "most": questions answered by the most-answered model}, best accuracy (sort="accuracy") or score (sort="score") first. S: the mean
 score (partial credit) in %. models: the ones to compare (default: every model in rows
-    with enough answers; models named here are compared whatever their count). memory: memory_gb()."""
+    with enough answers and a score of at least min_score, 0 for any; models named here are compared
+    whatever their count and score). memory: memory_gb()."""
     no_answers = sorted({r["model"] for r in rows} - {r["model"] for r in rows if not r.get("error")})
     rows = [r for r in rows if not r.get("error")]     # a server or connection error isn't an answer
     tests = compare.per_question(rows)
@@ -70,47 +74,70 @@ score (partial credit) in %. models: the ones to compare (default: every model i
         for m in t["cells"]:
             counts[m] = counts.get(m, 0) + 1
     most = max(counts.values(), default=0)
+    picked = models
     if models:
         no_answers = [m for m in no_answers if m in models]
         models, left_out = sorted(m for m in models if m in counts), []
     else:
         models = sorted(m for m in counts if counts[m] >= MIN_SHARE * most)
         left_out = sorted(((m, n) for m, n in counts.items() if m not in models), key=lambda x: (-x[1], x[0]))
-    shared = [t for t in tests if models and all(m in t["cells"] for m in models)]
-    out = []
-    for m in models:
-        cells = [t["cells"][m] for t in shared]
-        answers = sum(c["n"] for c in cells)
-        out.append({"model": m, "tokens": sum(c["tokens"] or 0 for c in cells),
-                    "total_s": sum(c["secs"] or 0 for c in cells),
-                    "passed": sum(c["passed"] for c in cells), "answers": answers,
-                    "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
-                    "S": 100 * sum(c["score"] * c["n"] for c in cells) / answers if answers else None,
-                    # failed only because max_tokens ran out (overthinking, not wrong yet)
-                    "cut": sum(c["truncated"] for c in cells),
-                    "cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
-                    # ran past the max_tokens sent: the provider didn't stop it, so it had more room than asked
-                    "over": sum(c.get("over", 0) for c in cells),
-                    # generation speed and time to first token over the same answers (medians)
-                    "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
-                    "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
-                    "memory_gb": (memory or {}).get(m),
-                    "repeats": (min(c["n"] for c in cells), max(c["n"] for c in cells)) if cells else None,
-                    # answers that ran while other models were served shared the machine's speed
-                    "alongside": any(r.get("alongside") for c in cells for r in c["rows"])})
-    fastest = min((x["total_s"] for x in out if x["total_s"] > 0), default=None)
-    fewest = min((x["tokens"] for x in out if x["tokens"] > 0), default=None)
-    for x in out:
-        x["P"], x["T"] = score(x["tokens"], fewest), score(x["total_s"], fastest)
-        # shown on the bars: how many times the leanest / fastest model's total (1.0 = the best)
-        x["tokens_x"] = x["tokens"] / fewest if fewest and x["tokens"] else None
-        x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
-        x["fewest"] = bool(fewest) and x["tokens"] == fewest
-        x["fastest"] = bool(fastest) and x["total_s"] == fastest
+
+    def compare_on(models):
+        shared = [t for t in tests if models and all(m in t["cells"] for m in models)]
+        out = []
+        for m in models:
+            cells = [t["cells"][m] for t in shared]
+            answers = sum(c["n"] for c in cells)
+            out.append({"model": m, "tokens": sum(c["tokens"] or 0 for c in cells),
+                        "total_s": sum(c["secs"] or 0 for c in cells),
+                        "passed": sum(c["passed"] for c in cells), "answers": answers,
+                        "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
+                        "S": 100 * sum(c["score"] * c["n"] for c in cells) / answers if answers else None,
+                        # failed only because max_tokens ran out (overthinking, not wrong yet)
+                        "cut": sum(c["truncated"] for c in cells),
+                        "cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
+                        # ran past the max_tokens sent: the provider didn't stop it, so it had more room than asked
+                        "over": sum(c.get("over", 0) for c in cells),
+                        # generation speed and time to first token over the same answers (medians)
+                        "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
+                        "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
+                        "memory_gb": (memory or {}).get(m),
+                        "repeats": (min(c["n"] for c in cells), max(c["n"] for c in cells)) if cells else None,
+                        # answers that ran while other models were served shared the machine's speed
+                        "alongside": any(r.get("alongside") for c in cells for r in c["rows"])})
+        fastest = min((x["total_s"] for x in out if x["total_s"] > 0), default=None)
+        fewest = min((x["tokens"] for x in out if x["tokens"] > 0), default=None)
+        for x in out:
+            x["P"], x["T"] = score(x["tokens"], fewest), score(x["total_s"], fastest)
+            # shown on the bars: how many times the leanest / fastest model's total (1.0 = the best)
+            x["tokens_x"] = x["tokens"] / fewest if fewest and x["tokens"] else None
+            x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
+            x["fewest"] = bool(fewest) and x["tokens"] == fewest
+            x["fastest"] = bool(fastest) and x["total_s"] == fastest
+        return shared, out
+
+    shared, out = compare_on(models)
+    low_score = []
+    if not picked and min_score:
+        # by default, models scoring below min_score are left out too (unless every model is), and the rest
+        # compared again: without them the shared questions can grow, and scores move, so until none is below
+        while True:
+            low = [x for x in out if _score(x) < min_score]
+            if not low or len(low) == len(out):
+                break
+            low_score += [(x["model"], _score(x)) for x in low]
+            models = [m for m in models if m not in {x["model"] for x in low}]
+            shared, out = compare_on(models)
+        low_score.sort(key=lambda x: (-x[1], x[0]))
     first, then = ("S", "A") if sort == "score" else ("A", "S")
     out.sort(key=lambda x: (-(x[first] or 0), -(x[then] or 0), -(x["T"] or 0), x["model"]))
-    return {"questions": len(shared), "models": out, "left_out": left_out, "no_answers": no_answers,
-            "most": most}
+    return {"questions": len(shared), "models": out, "left_out": left_out, "low_score": low_score,
+            "min_score": min_score, "no_answers": no_answers, "most": most}
+
+
+def _score(x):
+    """A model's score (partial credit, %), or its accuracy when there's none."""
+    return x["S"] if x.get("S") is not None else (x["A"] or 0)
 
 
 # (letter, name, which way is better, what the bar measures, the number beside it, colour). P and T bars
@@ -200,13 +227,17 @@ def bars(result, width=20, markup=True, click=None):
 
 
 def left_out_lines(result):
-    """Plain sentences on the models not compared (too few answers, or only server errors) and on
+    """Plain sentences on the models not compared (too few answers, a low score, or only server errors) and on
     answers that shared the machine with other models."""
     out = []
     if result["left_out"]:
         out.append("Left out (fewer than half as many answers as the most-answered model; pick them under "
                    "Models or --only to include them): "
                    + ", ".join(f"{m} ({n} of {result['most']})" for m, n in result["left_out"]))
+    if result.get("low_score"):
+        out.append(f"Left out (scored below {result.get('min_score', MIN_SCORE):g}%; lower the minimum score, or pick "
+                   "them under Models or --only, to include them): "
+                   + ", ".join(f"{m} ({compare.pct(v / 100)})" for m, v in result["low_score"]))
     if result["no_answers"]:
         out.append("No answers, only server errors: " + ", ".join(result["no_answers"]))
     shared = [x["model"] for x in result["models"] if x["alongside"]]
@@ -271,14 +302,14 @@ def scope_note(result, models):
                if any(x.get("over") for x in result["models"]) else ""))
 
 
-def for_engine(e, labels=None, packs=None, results_dir=None, sort="accuracy"):
+def for_engine(e, labels=None, packs=None, results_dir=None, sort="accuracy", min_score=MIN_SCORE):
     """(result, models) from a workspace's finished results (results on their way out excluded)."""
     paths = [p for p in compare.default_paths(results_dir or e.results_dir) if not e.superseded(p)]
     rows, infos = compare.load(paths) if paths else ([], [])
     if packs:
         rows = [r for r in rows if r["suite"] in packs]
     models = sorted({r["model"] for r in rows} & set(labels) if labels else {r["model"] for r in rows})
-    return index(rows, models if labels else None, memory_gb(infos), sort), models
+    return index(rows, models if labels else None, memory_gb(infos), sort, min_score), models
 
 
 def markdown(result, models):

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/src/tuieval/run_evals.py

@@ -405,6 +405,9 @@ def cmd_pta(argv):
     p.add_argument("--width", type=int, default=20, help="bar width in characters (default 20)")
     p.add_argument("--sort", choices=("accuracy", "score"), default="accuracy",
                    help="order models by answers right (default) or by score (partial credit)")
+    p.add_argument("--min-score", type=float, default=None,
+                   help="leave out models scoring below this %% (default 85; 0 shows every model; "
+                        "models named with --only are always shown)")
     p.add_argument("--models")
     p.add_argument("--results-dir")
     a = p.parse_args(argv)
@@ -413,7 +416,8 @@ def cmd_pta(argv):
     e = engine.Engine(models_path=a.models, results_dir=a.results_dir)
     labels = [x.strip() for x in a.only.split(",")] if a.only else None
     packs = [x.strip() for x in a.packs.split(",")] if a.packs else None
-    result, models = pta.for_engine(e, labels, packs, sort=a.sort)
+    result, models = pta.for_engine(e, labels, packs, sort=a.sort,
+                                    min_score=pta.MIN_SCORE if a.min_score is None else a.min_score)
     out = Console(highlight=False)
     if not models:
         sys.exit("no finished results to compare" + (" for those models or packs" if labels or packs else ""))

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/src/tuieval/tui.py

@@ -2934,6 +2934,7 @@ class ResultsScreen(Screen):
         self.smoke_dir = smoke_dir
 
     def compose(self) -> ComposeResult:
+        from . import pta
         yield Header()
         with Horizontal(id="results-top"):
             yield Static(id="results-info")
@@ -2945,6 +2946,9 @@ class ResultsScreen(Screen):
                     yield Select([("All packs", "")], id="pta-pack", allow_blank=False, value="")
                     yield Select([("By accuracy", "accuracy"), ("By score", "score")], id="pta-sort",
                                  allow_blank=False, value="accuracy")
+                    yield Select([("Any score", 0)] + [(f"Score ≥ {n}%", n) for n in pta.MIN_SCORES if n],
+                                 id="pta-min-score", allow_blank=False,
+                                 value=getattr(self.app, "pta_min_score", pta.MIN_SCORE))
                     yield Button("Models: all", id="pta-models")
                 yield Static("[dim]P parsimony (tokens): tokens used, reasoning and answer. T time: total time. "
                              "A accuracy: answers right. Verdicts and critical failures are in Production "
@@ -3275,6 +3279,13 @@ class ResultsScreen(Screen):
         out = insights.panels(rows, models, e.packs, ref, bases, by, engine=e if main else None)
         body.update("\n\n".join(insights.render(p, i) for i, p in enumerate(out, 1)))
 
+    @on(Select.Changed, "#pta-min-score")
+    def pta_min_score_changed(self, event):
+        """The minimum score: models below it are left out of the PTA index unless picked (kept for the session)."""
+        self.app.pta_min_score = event.value
+        if getattr(self, "rows", None) is not None:
+            self.fill_pta()
+
     @on(Select.Changed, "#pta-sort")
     def pta_sort_changed(self):
         """The Sort dropdown orders the bars and the table, taking over from a column sorted by a click."""
@@ -3351,9 +3362,10 @@ class ResultsScreen(Screen):
             self.query_one("#pta-bars", Static).update("")
             self.query_one("#pta-left-out", Static).update("")
             return
-        # none picked: every model with enough answers (pta.index leaves out the rest, and says so)
+        # none picked: every model with enough answers and score (pta.index leaves out the rest, and says so)
         result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos),
-                           self.query_one("#pta-sort", Select).value or "accuracy")
+                           self.query_one("#pta-sort", Select).value or "accuracy",
+                           self.query_one("#pta-min-score", Select).value or 0)
         self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
         self.pta_result = result
         removed = [m for m in self.pta_removed_now() if m in present]
@@ -3700,6 +3712,7 @@ class EvalsApp(App):
     #ins-body { height: 1fr; }
     #pta-controls Select { width: 28; margin-right: 1; }
     #pta-controls #pta-sort { width: 18; }
+    #pta-controls #pta-min-score { width: 18; }
     #pta-bars { height: auto; padding: 1 0 0 0; }
     #pta { width: 1fr; height: auto; margin-top: 1; }
     #pta-left-out { height: auto; padding: 1 0 0 0; }

{tuieval-0.2.0.dev141 → tuieval-0.2.0.dev143}/tests/test_tuieval.py

@@ -1691,7 +1691,7 @@ class PTAIndex(unittest.TestCase):
         rows = [row("fast", "q1", True, 2), row("fast", "q2", False, 2),
                 row("slow", "q1", True, 8), row("slow", "q2", True, 4), row("slow", "q2", True, 6, repeat=1),
                 row("slow", "q3", True, 100)]               # only slow answered q3: not compared
-        res = pta.index(rows)
+        res = pta.index(rows, min_score=0)    # fast scores 50%: kept
         self.assertEqual(res["questions"], 2)
         by = {x["model"]: x for x in res["models"]}
         self.assertEqual(by["fast"]["total_s"], 4)
@@ -1777,6 +1777,30 @@ class PTAIndex(unittest.TestCase):
         picked = pta.index(rows, ["a", "few"])      # picked by hand: compared anyway
         self.assertEqual((picked["questions"], picked["left_out"]), (1, []))
 
+    def test_low_scores_left_out_by_default(self):
+        from tuieval import pta
+
+        def row(model, test, ok):
+            return {"model": model, "suite": "p", "test": f"p: {test}", "ok": ok, "score": 1.0 if ok else 0.0,
+                    "latency": 1000, "tokens": 10, "truncated": False, "gen_tps": None, "repeat": 0}
+        rows = ([row("good", f"q{i}", True) for i in range(10)]
+                + [row("ok", f"q{i}", i != 0) for i in range(10)]                # 90%
+                + [row("weak", f"q{i}", i < 5) for i in range(8)])               # 62.5%, skipped q8 and q9
+        res = pta.index(rows)
+        self.assertEqual([x["model"] for x in res["models"]], ["good", "ok"])
+        self.assertEqual(res["questions"], 10)        # compared again without it: q8 and q9 count now
+        self.assertEqual(res["low_score"], [("weak", 62.5)])
+        self.assertIn("scored below 85%", "\n".join(pta.left_out_lines(res)))
+        self.assertIn("weak (62.5%)", "\n".join(pta.left_out_lines(res)))
+        picked = pta.index(rows, ["good", "weak"])     # picked by hand: shown whatever the score
+        self.assertEqual(([x["model"] for x in picked["models"]], picked["low_score"]), (["good", "weak"], []))
+        low = [r for r in rows if r["model"] != "good"]
+        self.assertEqual(pta.index([r for r in low if r["model"] == "weak"])["low_score"], [])   # all low: kept
+        self.assertEqual([x["model"] for x in pta.index(low)["models"]], ["ok"])
+        self.assertEqual([x["model"] for x in pta.index(rows, min_score=0)["models"]], ["good", "ok", "weak"])
+        self.assertEqual([x["model"] for x in pta.index(rows, min_score=95)["models"]], ["good"])
+        self.assertIn("scored below 95%", "\n".join(pta.left_out_lines(pta.index(rows, min_score=95))))
+
     def test_results_sort_values(self):
         from tuieval.tui import diff_badge, sort_cell, sort_key, sort_value
         got = [sort_value(c) for c in ("1,234 ★", "3.1k", "2.4×", "4m10s", "1h02m", "12.5s", diff_badge("hard"),
@@ -1964,7 +1988,8 @@ class PTAIndex(unittest.TestCase):
             finally:
                 right.stop()
                 wrong.stop()
-            out = tuieval(ws, "pta").stdout
+            self.assertIn("scored below 85%", tuieval(ws, "pta").stdout)     # hosted (0%) left out by default
+            out = tuieval(ws, "pta", "--min-score", "0").stdout
             self.assertIn("Compared on the 4 questions all 2 model(s) answered", out)
             bars = {l.split()[0]: l for l in out.splitlines() if "█" in l}
             self.assertEqual((bars["good"].count("1.0×"), "100%" in bars["good"]), (2, True))
@@ -1983,6 +2008,7 @@ class PTAIndex(unittest.TestCase):
                     app = EvalsApp({})
                     async with app.run_test(size=(180, 60)) as pilot:
                         await pilot.pause()
+                        app.pta_min_score = 0                                   # hosted scores 0%
                         app.push_screen(ResultsScreen())
                         await pilot.pause(0.5)
                         t = app.screen.query_one("#pta", DataTable)