tuieval 0.2.0.dev129__py3-none-any.whl → 0.2.0.dev131__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tuieval/_version.py +2 -2
- tuieval/compare.py +17 -3
- tuieval/insights.py +12 -6
- tuieval/pta.py +23 -9
- tuieval/run_evals.py +3 -1
- tuieval/tui.py +46 -9
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev131.dist-info}/METADATA +1 -1
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev131.dist-info}/RECORD +11 -11
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev131.dist-info}/WHEEL +0 -0
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev131.dist-info}/entry_points.txt +0 -0
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev131.dist-info}/licenses/LICENSE +0 -0
tuieval/_version.py
CHANGED
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.2.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 2, 0, '
|
|
21
|
+
__version__ = version = '0.2.0.dev131'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 2, 0, 'dev131')
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
tuieval/compare.py
CHANGED
|
@@ -351,10 +351,22 @@ def speed(rows, infos, machine=None, speeds=None):
|
|
|
351
351
|
return header, table, frontier
|
|
352
352
|
|
|
353
353
|
|
|
354
|
+
NEAR_MISS = 0.8 # a failed answer scoring at least this (e.g. 5 of 6 checks) is a near miss
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
def checks_passed(r):
|
|
358
|
+
"""(passed, total) checks of a failed code answer (from its stored checks, or its reason), else None."""
|
|
359
|
+
checks = (r.get("raw") or {}).get("checks")
|
|
360
|
+
if checks:
|
|
361
|
+
return sum(v is None for v in checks.values()), len(checks)
|
|
362
|
+
m = re.match(r"Failed (\d+)/(\d+) checks", r.get("reason") or "")
|
|
363
|
+
return (int(m[2]) - int(m[1]), int(m[2])) if m else None
|
|
364
|
+
|
|
365
|
+
|
|
354
366
|
def per_question(rows):
|
|
355
367
|
"""[{test, name, suite, difficulty, cells: {model: cell}, spread}] with one cell per model that
|
|
356
|
-
answered: n answers, passes, median tokens / seconds / tok/s over repeats,
|
|
357
|
-
partial (some answers from a pack still running), rows (the answers, for the detail view).
|
|
368
|
+
answered: n answers, passes, mean score (partial credit), median tokens / seconds / tok/s over repeats,
|
|
369
|
+
truncated count, partial (some answers from a pack still running), rows (the answers, for the detail view).
|
|
358
370
|
spread = {"tokens": .., "secs": ..}: largest / smallest model median (1 if one model)."""
|
|
359
371
|
tests = {}
|
|
360
372
|
for r in rows:
|
|
@@ -369,6 +381,7 @@ def per_question(rows):
|
|
|
369
381
|
c.update(rows=v, n=len(v), passed=sum(r["ok"] for r in v), truncated=sum(r["truncated"] for r in v),
|
|
370
382
|
over=sum(bool(r.get("over_budget")) for r in v),
|
|
371
383
|
partial=any(r.get("partial") for r in v),
|
|
384
|
+
score=statistics.fmean(r.get("score", r["ok"]) for r in v),
|
|
372
385
|
tokens=median(r["tokens"] or None for r in v), secs=median(r["latency"] / 1000 or None for r in v),
|
|
373
386
|
tps=median(r["gen_tps"] for r in v))
|
|
374
387
|
t["spread"] = _spread(t["cells"])
|
|
@@ -404,7 +417,7 @@ def complete_models(rows, suite, test_ids=None):
|
|
|
404
417
|
def compare_models(tests, models):
|
|
405
418
|
"""Per model over per_question entries that all `models` answered: passes, token and time
|
|
406
419
|
totals/medians (totals add each question's median over repeats, so repeats don't inflate them), questions where it used the fewest tokens / was fastest (among passing
|
|
407
|
-
answers for tokens; ties count for each), answers cut off."""
|
|
420
|
+
answers for tokens; ties count for each), answers cut off, mean score (partial credit)."""
|
|
408
421
|
out = []
|
|
409
422
|
for m in models:
|
|
410
423
|
cs = [t["cells"][m] for t in tests]
|
|
@@ -421,6 +434,7 @@ def compare_models(tests, models):
|
|
|
421
434
|
fastest += 1
|
|
422
435
|
rs = [r for c in cs for r in c["rows"]]
|
|
423
436
|
out.append({"model": m, "n": len(cs), "passed": passed, "total_tokens": total,
|
|
437
|
+
"score": statistics.fmean(c["score"] for c in cs) if cs else None,
|
|
424
438
|
"avg_pp_tps": _rate([(r.get("prompt_tokens"), r["prompt_tps"]) for r in rs]),
|
|
425
439
|
"avg_gen_tps": _rate([(r["tokens"], r["gen_tps"]) for r in rs]),
|
|
426
440
|
"med_tokens": median(c["tokens"] for c in cs), "med_secs": median(c["secs"] for c in cs),
|
tuieval/insights.py
CHANGED
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
2 use cases accuracy per model and use case (pack group) as a heatmap; the best marked
|
|
7
7
|
3 vs base a fine-tune against its base model (models.toml `base`): per pack, and the question
|
|
8
8
|
categories it gained or lost most
|
|
9
|
-
4 why they fail each model's answers split into right,
|
|
10
|
-
crashed or hung, and
|
|
9
|
+
4 why they fail each model's answers split into right, near miss (most checks passed), partly right,
|
|
10
|
+
wrong, cut off (or out of time), no answer / bad format, code crashed or hung, and
|
|
11
|
+
critical failures
|
|
11
12
|
5 accuracy / cost accuracy against median tokens (or seconds) per question; the models no other beats
|
|
12
13
|
on both
|
|
13
14
|
6 questions questions every model fails (check them), every model passes (no signal), and the
|
|
@@ -267,7 +268,8 @@ def categories(rows, m, base, min_q=2):
|
|
|
267
268
|
|
|
268
269
|
|
|
269
270
|
# ---------------------------------------------------------------- 4 why they fail
|
|
270
|
-
KINDS = (("right", "green", "█"), ("
|
|
271
|
+
KINDS = (("right", "green", "█"), ("near miss", "#87af5f", "█"), ("partly right", "#af875f", "█"),
|
|
272
|
+
("wrong", "red", "█"), ("cut off", "yellow", "█"),
|
|
271
273
|
("no answer / format", "magenta", "█"), ("code crashed or hung", "cyan", "█"), ("critical", "#ff5faf", "▓"))
|
|
272
274
|
FORMAT = ("No 'ANSWER:' line", "Empty answer", "No code", "no tool call")
|
|
273
275
|
CRASHED = ("Timed out after", "Harness crashed", "grader error")
|
|
@@ -278,13 +280,15 @@ def kind(r):
|
|
|
278
280
|
return "right"
|
|
279
281
|
if r.get("severity") == "critical":
|
|
280
282
|
return "critical"
|
|
281
|
-
if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET")):
|
|
283
|
+
if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET", "time limit")):
|
|
282
284
|
return "cut off"
|
|
283
285
|
if r["reason"].startswith(FORMAT):
|
|
284
286
|
return "no answer / format"
|
|
285
287
|
if r["reason"].startswith(CRASHED):
|
|
286
288
|
return "code crashed or hung"
|
|
287
|
-
|
|
289
|
+
# partial credit (e.g. checks passed): close isn't the same as all wrong
|
|
290
|
+
score = r.get("score") or 0
|
|
291
|
+
return "near miss" if score >= compare.NEAR_MISS else "partly right" if score > 0 else "wrong"
|
|
288
292
|
|
|
289
293
|
|
|
290
294
|
def _split(counts, total, width):
|
|
@@ -322,7 +326,9 @@ def failure_mix(rows, models, width=36):
|
|
|
322
326
|
top, n = max(((k, c[k]) for k, *_ in KINDS[1:-1]), key=lambda x: x[1])
|
|
323
327
|
if n / fails >= 0.4 and top != "wrong":
|
|
324
328
|
actions.append(f"{m}: {n} of its {fails} failures are {top}: " + {
|
|
325
|
-
"cut off": "it thinks too long for max_tokens, rather than reasoning wrong.",
|
|
329
|
+
"cut off": "it thinks too long for max_tokens (or the time limit), rather than reasoning wrong.",
|
|
330
|
+
"near miss": "they pass most of their checks: it's closer than its accuracy shows.",
|
|
331
|
+
"partly right": "they pass some of their checks, not none.",
|
|
326
332
|
"no answer / format": "it doesn't follow the answer format (check its chat template and the "
|
|
327
333
|
"pack's instructions).",
|
|
328
334
|
"code crashed or hung": "its code crashes or never finishes."}[top])
|
tuieval/pta.py
CHANGED
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
model's ("28×").
|
|
8
8
|
A accuracy the share of answers to those questions that passed (%).
|
|
9
9
|
|
|
10
|
+
Beside A, the score: partial credit averaged over the same answers (e.g. a code answer passing 5 of 6
|
|
11
|
+
checks scores 83%, where A counts it wrong). Shown when it differs from A; models can be ordered by it.
|
|
12
|
+
|
|
10
13
|
The P and T bars are as long as the tokens and time used (the model that used the most fills its bar,
|
|
11
14
|
the others in proportion, at least one block), so shorter is better; the A bar: longer is better.
|
|
12
15
|
P and T also carry a 0-100 score (100 for the best, POINTS_PER_DOUBLING points less for every
|
|
@@ -46,11 +49,15 @@ def memory_gb(infos):
|
|
|
46
49
|
return out
|
|
47
50
|
|
|
48
51
|
|
|
49
|
-
|
|
50
|
-
|
|
52
|
+
SORTS = ("accuracy", "score")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def index(rows, models=None, memory=None, sort="accuracy"):
|
|
56
|
+
"""{"questions": n shared, "models": [{model, P, T, A, S, cut, cut_pct, tokens, total_s, passed, answers, alongside,
|
|
51
57
|
fewest, fastest, tps, ttft, memory_gb}],
|
|
52
58
|
"left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
|
|
53
|
-
most-answered model}, best accuracy
|
|
59
|
+
most-answered model}, best accuracy (sort="accuracy") or score (sort="score") first. S: the mean
|
|
60
|
+
score (partial credit) in %. models: the ones to compare (default: every model in rows
|
|
54
61
|
with enough answers; models named here are compared whatever their count). memory: memory_gb()."""
|
|
55
62
|
no_answers = sorted({r["model"] for r in rows} - {r["model"] for r in rows if not r.get("error")})
|
|
56
63
|
rows = [r for r in rows if not r.get("error")] # a server or connection error isn't an answer
|
|
@@ -75,6 +82,7 @@ def index(rows, models=None, memory=None):
|
|
|
75
82
|
"total_s": sum(c["secs"] or 0 for c in cells),
|
|
76
83
|
"passed": sum(c["passed"] for c in cells), "answers": answers,
|
|
77
84
|
"A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
|
|
85
|
+
"S": 100 * sum(c["score"] * c["n"] for c in cells) / answers if answers else None,
|
|
78
86
|
# failed only because max_tokens ran out (overthinking, not wrong yet)
|
|
79
87
|
"cut": sum(c["truncated"] for c in cells),
|
|
80
88
|
"cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
|
|
@@ -95,7 +103,8 @@ def index(rows, models=None, memory=None):
|
|
|
95
103
|
x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
|
|
96
104
|
x["fewest"] = bool(fewest) and x["tokens"] == fewest
|
|
97
105
|
x["fastest"] = bool(fastest) and x["total_s"] == fastest
|
|
98
|
-
|
|
106
|
+
first, then = ("S", "A") if sort == "score" else ("A", "S")
|
|
107
|
+
out.sort(key=lambda x: (-(x[first] or 0), -(x[then] or 0), -(x["T"] or 0), x["model"]))
|
|
99
108
|
return {"questions": len(shared), "models": out, "left_out": left_out, "no_answers": no_answers,
|
|
100
109
|
"most": most}
|
|
101
110
|
|
|
@@ -145,6 +154,10 @@ def bars(result, width=20, markup=True, click=None):
|
|
|
145
154
|
if v is not None and v < 100: # full only when every answer passed
|
|
146
155
|
full = min(full, width - 1)
|
|
147
156
|
num = "-" if v is None else compare.pct(v / 100)
|
|
157
|
+
score = x.get("S")
|
|
158
|
+
if score is not None and v is not None and compare.pct(score / 100) != num:
|
|
159
|
+
note = f"score {compare.pct(score / 100)}"
|
|
160
|
+
num = f"{num:>6} " + (f"[dim]{note}[/dim]" if markup else note)
|
|
148
161
|
cut = x.get("cut_pct") or 0
|
|
149
162
|
part = min(width - full, max(1, round(width * cut / 100))) if cut else 0
|
|
150
163
|
if cut: # after the accuracy, which stays aligned with the other rows'
|
|
@@ -189,11 +202,12 @@ def table(result):
|
|
|
189
202
|
"""(header, rows) for a plain table of the index."""
|
|
190
203
|
n = result["questions"] or 1
|
|
191
204
|
over = any(x.get("over") for x in result["models"]) # a column only when some answer ran past max_tokens
|
|
192
|
-
header = ["model", "tokens/answer", "time/answer", "answers right", "truncated", *(["over budget"] if over else []),
|
|
205
|
+
header = ["model", "tokens/answer", "time/answer", "answers right", "score", "truncated", *(["over budget"] if over else []),
|
|
193
206
|
"tok/s", "TTFT s", "memory GB"]
|
|
194
207
|
rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
|
|
195
208
|
fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
|
|
196
|
-
f"{x['passed']}/{x['answers']}",
|
|
209
|
+
f"{x['passed']}/{x['answers']}", "-" if x.get("S") is None else compare.pct(x["S"] / 100),
|
|
210
|
+
f"{x.get('cut') or 0}/{x['answers']}",
|
|
197
211
|
*([f"{x.get('over') or 0}/{x['answers']}"] if over else []),
|
|
198
212
|
_num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
|
|
199
213
|
_num(x.get("memory_gb"), ".1f")] for x in result["models"]]
|
|
@@ -228,7 +242,7 @@ def scope_note(result, models):
|
|
|
228
242
|
return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
|
|
229
243
|
"P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
|
|
230
244
|
"and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
|
|
231
|
-
"(longer is better)"
|
|
245
|
+
"(longer is better); score: partial credit, e.g. 5 of 6 checks passed is 83% (shown when it differs)"
|
|
232
246
|
+ ("; orange, (n% truncated): answers cut off at max_tokens. They count as wrong, but the model ran "
|
|
233
247
|
"out of tokens rather than answering wrong." if any(x.get("cut") for x in result["models"]) else ".")
|
|
234
248
|
+ (" Over budget: answers that used more tokens than the max_tokens sent, because the provider didn't "
|
|
@@ -236,14 +250,14 @@ def scope_note(result, models):
|
|
|
236
250
|
if any(x.get("over") for x in result["models"]) else ""))
|
|
237
251
|
|
|
238
252
|
|
|
239
|
-
def for_engine(e, labels=None, packs=None, results_dir=None):
|
|
253
|
+
def for_engine(e, labels=None, packs=None, results_dir=None, sort="accuracy"):
|
|
240
254
|
"""(result, models) from a workspace's finished results (results on their way out excluded)."""
|
|
241
255
|
paths = [p for p in compare.default_paths(results_dir or e.results_dir) if not e.superseded(p)]
|
|
242
256
|
rows, infos = compare.load(paths) if paths else ([], [])
|
|
243
257
|
if packs:
|
|
244
258
|
rows = [r for r in rows if r["suite"] in packs]
|
|
245
259
|
models = sorted({r["model"] for r in rows} & set(labels) if labels else {r["model"] for r in rows})
|
|
246
|
-
return index(rows, models if labels else None, memory_gb(infos)), models
|
|
260
|
+
return index(rows, models if labels else None, memory_gb(infos), sort), models
|
|
247
261
|
|
|
248
262
|
|
|
249
263
|
def markdown(result, models):
|
tuieval/run_evals.py
CHANGED
|
@@ -402,6 +402,8 @@ def cmd_pta(argv):
|
|
|
402
402
|
p.add_argument("--only", help="comma-separated model labels (default: every model with results)")
|
|
403
403
|
p.add_argument("--packs", help="comma-separated packs (default: all)")
|
|
404
404
|
p.add_argument("--width", type=int, default=20, help="bar width in characters (default 20)")
|
|
405
|
+
p.add_argument("--sort", choices=("accuracy", "score"), default="accuracy",
|
|
406
|
+
help="order models by answers right (default) or by score (partial credit)")
|
|
405
407
|
p.add_argument("--models")
|
|
406
408
|
p.add_argument("--results-dir")
|
|
407
409
|
a = p.parse_args(argv)
|
|
@@ -410,7 +412,7 @@ def cmd_pta(argv):
|
|
|
410
412
|
e = engine.Engine(models_path=a.models, results_dir=a.results_dir)
|
|
411
413
|
labels = [x.strip() for x in a.only.split(",")] if a.only else None
|
|
412
414
|
packs = [x.strip() for x in a.packs.split(",")] if a.packs else None
|
|
413
|
-
result, models = pta.for_engine(e, labels, packs)
|
|
415
|
+
result, models = pta.for_engine(e, labels, packs, sort=a.sort)
|
|
414
416
|
out = Console(highlight=False)
|
|
415
417
|
if not models:
|
|
416
418
|
sys.exit("no finished results to compare" + (" for those models or packs" if labels or packs else ""))
|
tuieval/tui.py
CHANGED
|
@@ -2892,6 +2892,10 @@ class ResultsScreen(Screen):
|
|
|
2892
2892
|
def action_runs(self):
|
|
2893
2893
|
self.app.open_sessions()
|
|
2894
2894
|
|
|
2895
|
+
FAIL_KINDS = [("Every kind", ""), ("Near misses", "near miss"), ("Partly right", "partly right"),
|
|
2896
|
+
("Wrong", "wrong"), ("Cut off", "cut off"), ("No answer / format", "no answer / format"),
|
|
2897
|
+
("Code crashed or hung", "code crashed or hung"), ("Critical", "critical")]
|
|
2898
|
+
|
|
2895
2899
|
READINESS_HELP = ("[dim]PASS needs a Certify run with zero critical failures over enough trials and an "
|
|
2896
2900
|
"accuracy lower bound that clears the pack's gate (pack.toml). Screening can only FAIL "
|
|
2897
2901
|
"or look promising.[/dim]")
|
|
@@ -2913,6 +2917,8 @@ class ResultsScreen(Screen):
|
|
|
2913
2917
|
with TabPane("PTA index", id="tab-pta"):
|
|
2914
2918
|
with Horizontal(id="pta-controls"):
|
|
2915
2919
|
yield Select([("All packs", "")], id="pta-pack", allow_blank=False, value="")
|
|
2920
|
+
yield Select([("By accuracy", "accuracy"), ("By score", "score")], id="pta-sort",
|
|
2921
|
+
allow_blank=False, value="accuracy")
|
|
2916
2922
|
yield Button("Models: all", id="pta-models")
|
|
2917
2923
|
yield Static("[dim]P parsimony (tokens): tokens used, reasoning and answer. T time: total time. "
|
|
2918
2924
|
"A accuracy: answers right. Verdicts and critical failures are in Production "
|
|
@@ -2954,6 +2960,7 @@ class ResultsScreen(Screen):
|
|
|
2954
2960
|
with TabPane("Failures", id="tab-failures"):
|
|
2955
2961
|
with Horizontal(id="fail-controls"):
|
|
2956
2962
|
yield Button("Models: all", id="fail-models")
|
|
2963
|
+
yield Select(self.FAIL_KINDS, id="fail-kind", allow_blank=False, value="")
|
|
2957
2964
|
yield Static(id="fail-count")
|
|
2958
2965
|
yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
|
|
2959
2966
|
yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
|
|
@@ -3045,24 +3052,40 @@ class ResultsScreen(Screen):
|
|
|
3045
3052
|
self.sort_tables()
|
|
3046
3053
|
|
|
3047
3054
|
def fill_failures(self):
|
|
3048
|
-
"""Every wrong answer, of the models picked (Models: shared with the other tabs)
|
|
3055
|
+
"""Every wrong answer, of the models picked (Models: shared with the other tabs), of the kind picked
|
|
3056
|
+
(insights.kind: near miss, partly right, wrong, cut off, …)."""
|
|
3057
|
+
from . import insights
|
|
3049
3058
|
f = self.query_one("#failures", DataTable)
|
|
3050
3059
|
f.clear(columns=True)
|
|
3051
|
-
f.add_columns("Model", "Level", "Test", "Answered", "Reason")
|
|
3060
|
+
f.add_columns("Model", "Level", "Test", "Answered", "Kind", "Reason")
|
|
3052
3061
|
every = {r["model"] for r in self.rows}
|
|
3053
3062
|
chosen = {m for m in self.app.pq_models if m in every}
|
|
3054
|
-
|
|
3063
|
+
wanted = self.query_one("#fail-kind", Select).value or ""
|
|
3064
|
+
rows = [r for r in self.rows if (not chosen or r["model"] in chosen)
|
|
3065
|
+
and (r["ok"] or not wanted or insights.kind(r) == wanted)]
|
|
3055
3066
|
# compare.failures lists the failed rows in order, so they line up with fail_rows
|
|
3056
3067
|
self.fail_rows = [r for r in rows if not r["ok"]]
|
|
3068
|
+
colors = {k: c for k, c, _ in insights.KINDS}
|
|
3057
3069
|
for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
|
|
3058
|
-
|
|
3059
|
-
|
|
3070
|
+
r = self.fail_rows[i]
|
|
3071
|
+
when = dates.day(dates.answered_at(r.get("raw") or {}))
|
|
3072
|
+
k = insights.kind(r)
|
|
3073
|
+
checks = compare.checks_passed(r)
|
|
3074
|
+
label = f"{k} {checks[0]}/{checks[1]}" if checks and k in ("near miss", "partly right", "wrong") else k
|
|
3075
|
+
f.add_row(model, diff_badge(level), rich_escape(test), when, f"[{colors[k]}]{label}[/]",
|
|
3076
|
+
rich_escape(reason[:130]), key=str(i))
|
|
3060
3077
|
self.query_one("#fail-models", Button).label = self.models_label(chosen)
|
|
3061
3078
|
n = len(self.fail_rows)
|
|
3062
3079
|
self.query_one("#fail-count", Static).update(
|
|
3063
|
-
f"{n}
|
|
3080
|
+
f"{n} {'' if not wanted else wanted + ' '}failure{'s' * (n != 1)}"
|
|
3081
|
+
+ (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
|
|
3064
3082
|
self.sort_table(f)
|
|
3065
3083
|
|
|
3084
|
+
@on(Select.Changed, "#fail-kind")
|
|
3085
|
+
def fail_kind_changed(self):
|
|
3086
|
+
if getattr(self, "rows", None) is not None:
|
|
3087
|
+
self.fill_failures()
|
|
3088
|
+
|
|
3066
3089
|
@on(DataTable.RowSelected, "#failures")
|
|
3067
3090
|
def failure_selected(self, event):
|
|
3068
3091
|
# in the order shown, so [ and ] step through them as sorted
|
|
@@ -3223,6 +3246,13 @@ class ResultsScreen(Screen):
|
|
|
3223
3246
|
out = insights.panels(rows, models, e.packs, ref, bases, by, engine=e if main else None)
|
|
3224
3247
|
body.update("\n\n".join(insights.render(p, i) for i, p in enumerate(out, 1)))
|
|
3225
3248
|
|
|
3249
|
+
@on(Select.Changed, "#pta-sort")
|
|
3250
|
+
def pta_sort_changed(self):
|
|
3251
|
+
"""The Sort dropdown orders the bars and the table, taking over from a column sorted by a click."""
|
|
3252
|
+
self.app.result_sorts.pop("pta", None)
|
|
3253
|
+
if getattr(self, "rows", None) is not None:
|
|
3254
|
+
self.fill_pta()
|
|
3255
|
+
|
|
3226
3256
|
@on(Select.Changed, "#pta-pack")
|
|
3227
3257
|
@on(Select.Changed, "#pq-pack")
|
|
3228
3258
|
def pack_changed(self, event):
|
|
@@ -3293,7 +3323,8 @@ class ResultsScreen(Screen):
|
|
|
3293
3323
|
self.query_one("#pta-left-out", Static).update("")
|
|
3294
3324
|
return
|
|
3295
3325
|
# none picked: every model with enough answers (pta.index leaves out the rest, and says so)
|
|
3296
|
-
result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos)
|
|
3326
|
+
result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos),
|
|
3327
|
+
self.query_one("#pta-sort", Select).value or "accuracy")
|
|
3297
3328
|
self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
|
|
3298
3329
|
self.pta_result = result
|
|
3299
3330
|
removed = [m for m in self.pta_removed_now() if m in present]
|
|
@@ -3439,7 +3470,8 @@ class ResultsScreen(Screen):
|
|
|
3439
3470
|
tt = r["total_secs"]
|
|
3440
3471
|
cells.append([r["model"],
|
|
3441
3472
|
hi(tt, ("-" if not tt else f"{tt:.1f}s" if tt < 60 else fmt_secs(tt)), "total_secs", "min"),
|
|
3442
|
-
hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max")
|
|
3473
|
+
hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max")
|
|
3474
|
+
+ (f" [dim]{r['score']:.0%}[/dim]" if r["passed"] < r["n"] and r["score"] else ""),
|
|
3443
3475
|
hi(r["med_tokens"], f"{r['med_tokens']:,.0f}" if r["med_tokens"] else "-", "med_tokens", "min"),
|
|
3444
3476
|
hi(r["total_tokens"], f"{r['total_tokens']:,.0f}", "total_tokens", "min"),
|
|
3445
3477
|
hi(r["tokens_per_pass"], f"{r['tokens_per_pass']:,.0f}" if r["tokens_per_pass"] else "-",
|
|
@@ -3469,7 +3501,7 @@ class ResultsScreen(Screen):
|
|
|
3469
3501
|
"shared the machine's speed: compare their times with care.[/yellow]")
|
|
3470
3502
|
title.update(line + " [dim]Time / Tokens: each question's median, added up. Med tok / Med s: median "
|
|
3471
3503
|
"per answer. Tok/✓: tokens per answer right. Prompt t/s / Gen t/s: all tokens over all the time "
|
|
3472
|
-
"they took (cached prompts left out). Fewest / Fastest: questions won; ties count for both. "
|
|
3504
|
+
"they took (cached prompts left out). Passed: questions right on every repeat, then the score (partial credit, e.g. checks passed) when some failed. Fewest / Fastest: questions won; ties count for both. "
|
|
3473
3505
|
"Cut off: answers that hit max_tokens.[/dim]")
|
|
3474
3506
|
|
|
3475
3507
|
@staticmethod
|
|
@@ -3478,6 +3510,9 @@ class ResultsScreen(Screen):
|
|
|
3478
3510
|
else f"[yellow]{c['passed']}/{c['n']}[/yellow]")
|
|
3479
3511
|
if c["n"] > 1 and c["passed"] in (0, c["n"]):
|
|
3480
3512
|
mark += f"[dim]{c['n']}[/dim]"
|
|
3513
|
+
if c["passed"] < c["n"] and c.get("score"): # partial credit: how close it came
|
|
3514
|
+
checks = compare.checks_passed(c["rows"][0]) if c["n"] == 1 else None
|
|
3515
|
+
mark += f" [yellow]{checks[0]}/{checks[1]}[/yellow]" if checks else f" [yellow]{c['score']:.0%}[/yellow]"
|
|
3481
3516
|
tok = c["tokens"]
|
|
3482
3517
|
tok_s = "-" if not tok else (f"{tok / 1000:.1f}k" if tok >= 1000 else f"{tok:.0f}")
|
|
3483
3518
|
if tok and best and tok == best and c["passed"]:
|
|
@@ -3624,6 +3659,7 @@ class EvalsApp(App):
|
|
|
3624
3659
|
#ins-controls #ins-cost { width: 20; }
|
|
3625
3660
|
#ins-body { height: 1fr; }
|
|
3626
3661
|
#pta-controls Select { width: 28; margin-right: 1; }
|
|
3662
|
+
#pta-controls #pta-sort { width: 18; }
|
|
3627
3663
|
#pta-bars { height: auto; padding: 1 0 0 0; }
|
|
3628
3664
|
#pta { width: 1fr; height: auto; margin-top: 1; }
|
|
3629
3665
|
#pta-left-out { height: auto; padding: 1 0 0 0; }
|
|
@@ -3632,6 +3668,7 @@ class EvalsApp(App):
|
|
|
3632
3668
|
#pq-models { margin-right: 1; min-width: 20; }
|
|
3633
3669
|
#fail-controls { height: 3; }
|
|
3634
3670
|
#fail-models { margin-right: 1; min-width: 20; }
|
|
3671
|
+
#fail-kind { width: 28; margin-right: 1; }
|
|
3635
3672
|
#fail-count { width: auto; padding: 1 1; }
|
|
3636
3673
|
#pq-compare-title { padding: 0 1; height: auto; }
|
|
3637
3674
|
#pq-summary { height: auto; } /* every picked model, no scrolling: the tab scrolls instead */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tuieval
|
|
3
|
-
Version: 0.2.0.
|
|
3
|
+
Version: 0.2.0.dev131
|
|
4
4
|
Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
|
|
5
5
|
Project-URL: Homepage, https://github.com/ashe-wb/tuieval
|
|
6
6
|
Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
|
|
2
2
|
tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
-
tuieval/_version.py,sha256=
|
|
3
|
+
tuieval/_version.py,sha256=biCEvo8zrqcKekhkznGrBJyvEyPtmcQdRlZacVAu4JU,537
|
|
4
4
|
tuieval/agents.py,sha256=XmY4d4PAnC7BQcs_6KO9WTuIdpnwYVAdyhqItybWdAM,15408
|
|
5
5
|
tuieval/builtin.toml,sha256=romPxP9o3r_4z3V8bdsmE9oUk0G2plDeqygPbqrwOys,4529
|
|
6
6
|
tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
|
|
7
7
|
tuieval/client.py,sha256=5wo0ySMD3o8up3cxKZicUWYFvOLZB60kYBV7yxP6LjI,12378
|
|
8
|
-
tuieval/compare.py,sha256=
|
|
8
|
+
tuieval/compare.py,sha256=R_lGNSGPuDQ0j7-zuuniJtaRmI2KqDcDAhH82QO8Epc,36959
|
|
9
9
|
tuieval/dates.py,sha256=CVazlwNjCRhD__D12zXLdeqF2ySt15X3xLUQeE-o6PY,6293
|
|
10
10
|
tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
|
|
11
11
|
tuieval/engine.py,sha256=M78wy43ZL4qmwUaZBDhpRZH5dWQKd-o0z59kWg7waCQ,136050
|
|
12
12
|
tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
|
|
13
|
-
tuieval/insights.py,sha256=
|
|
13
|
+
tuieval/insights.py,sha256=BDOn9UkDUkCKFcG2YIfZcnECgIFIUHxQawPRROGvDpM,29534
|
|
14
14
|
tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
|
|
15
15
|
tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
|
|
16
16
|
tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
|
|
17
17
|
tuieval/packs.py,sha256=KOj0BkwsB2124aslGyauZonyyGqtSrCJ-WAozEntmqE,17466
|
|
18
18
|
tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
|
|
19
|
-
tuieval/pta.py,sha256=
|
|
19
|
+
tuieval/pta.py,sha256=Clfn9OBm85rHH1Ko7SAR7kIqhQdXhpEVN1nUhjVLTqQ,15480
|
|
20
20
|
tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
|
|
21
|
-
tuieval/run_evals.py,sha256=
|
|
21
|
+
tuieval/run_evals.py,sha256=afZIzSdZPSCWnhc9qV_L1TXexl7YgSF6eKlV7RimMao,50679
|
|
22
22
|
tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
|
|
23
23
|
tuieval/selftest.py,sha256=cIGSDCrCK-QISAXjKSJDB4sD-WwS32bn3Gn4Y0oGoF8,11502
|
|
24
|
-
tuieval/tui.py,sha256=
|
|
24
|
+
tuieval/tui.py,sha256=XUxx7Z_b5ekfL0qdkQg4DD4Tra5VUObNvoORCA8mAhc,204245
|
|
25
25
|
tuieval/tune.py,sha256=izz1Qs-d9PrWAqEZpyrLxfrlhlaouif9uFQSeJveL0k,41294
|
|
26
26
|
tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
|
|
27
27
|
tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
|
|
@@ -50,8 +50,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
|
|
|
50
50
|
tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
|
|
51
51
|
tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
|
|
52
52
|
tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
|
|
53
|
-
tuieval-0.2.0.
|
|
54
|
-
tuieval-0.2.0.
|
|
55
|
-
tuieval-0.2.0.
|
|
56
|
-
tuieval-0.2.0.
|
|
57
|
-
tuieval-0.2.0.
|
|
53
|
+
tuieval-0.2.0.dev131.dist-info/METADATA,sha256=Ok0YKjM61vA_ePXndEvbpY70zYpLJqQUqx5jGPdbV3c,15105
|
|
54
|
+
tuieval-0.2.0.dev131.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
55
|
+
tuieval-0.2.0.dev131.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
|
|
56
|
+
tuieval-0.2.0.dev131.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
|
|
57
|
+
tuieval-0.2.0.dev131.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|