tuieval 0.2.0.dev129__py3-none-any.whl → 0.2.0.dev133__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tuieval/_version.py CHANGED
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.2.0.dev129'
22
- __version_tuple__ = version_tuple = (0, 2, 0, 'dev129')
21
+ __version__ = version = '0.2.0.dev133'
22
+ __version_tuple__ = version_tuple = (0, 2, 0, 'dev133')
23
23
 
24
24
  __commit_id__ = commit_id = None
tuieval/compare.py CHANGED
@@ -248,12 +248,14 @@ def scorecard(rows, rng=None):
248
248
  suites = sorted({r["suite"] for r in rows})
249
249
  extra = dict(graders.COLUMNS) # columns your graders add (graders.scorecard_column)
250
250
  over = any(r.get("over_budget") for r in rows) # a column only when some answer ran past max_tokens
251
- header = ["model"] + suites + ["overall", "95% CI", "mean score", "all repeats", *extra, "errors", "trunc",
251
+ header = ["model", "repeats"] + suites + ["overall", "95% CI", "mean score", "all repeats", *extra, "errors", "trunc",
252
252
  *(["over budget"] if over else []), "rep"]
253
253
  table = []
254
254
  for m in models:
255
255
  mine = [r for r in rows if r["model"] == m]
256
- line = [m]
256
+ per_test = collections.Counter(r["test"] for r in mine) # answers per question: the repeats
257
+ few, most = min(per_test.values()), max(per_test.values())
258
+ line = [m, f"×{few}" if few == most else f"×{few}-{most}"]
257
259
  for s in suites:
258
260
  sub = [r for r in mine if r["suite"] == s]
259
261
  line.append(f"{sum(r['ok'] for r in sub)}/{len(sub)}" if sub else "-")
@@ -351,10 +353,22 @@ def speed(rows, infos, machine=None, speeds=None):
351
353
  return header, table, frontier
352
354
 
353
355
 
356
+ NEAR_MISS = 0.8 # a failed answer scoring at least this (e.g. 5 of 6 checks) is a near miss
357
+
358
+
359
+ def checks_passed(r):
360
+ """(passed, total) checks of a failed code answer (from its stored checks, or its reason), else None."""
361
+ checks = (r.get("raw") or {}).get("checks")
362
+ if checks:
363
+ return sum(v is None for v in checks.values()), len(checks)
364
+ m = re.match(r"Failed (\d+)/(\d+) checks", r.get("reason") or "")
365
+ return (int(m[2]) - int(m[1]), int(m[2])) if m else None
366
+
367
+
354
368
  def per_question(rows):
355
369
  """[{test, name, suite, difficulty, cells: {model: cell}, spread}] with one cell per model that
356
- answered: n answers, passes, median tokens / seconds / tok/s over repeats, truncated count,
357
- partial (some answers from a pack still running), rows (the answers, for the detail view).
370
+ answered: n answers, passes, mean score (partial credit), median tokens / seconds / tok/s over repeats,
371
+ truncated count, partial (some answers from a pack still running), rows (the answers, for the detail view).
358
372
  spread = {"tokens": .., "secs": ..}: largest / smallest model median (1 if one model)."""
359
373
  tests = {}
360
374
  for r in rows:
@@ -369,6 +383,7 @@ def per_question(rows):
369
383
  c.update(rows=v, n=len(v), passed=sum(r["ok"] for r in v), truncated=sum(r["truncated"] for r in v),
370
384
  over=sum(bool(r.get("over_budget")) for r in v),
371
385
  partial=any(r.get("partial") for r in v),
386
+ score=statistics.fmean(r.get("score", r["ok"]) for r in v),
372
387
  tokens=median(r["tokens"] or None for r in v), secs=median(r["latency"] / 1000 or None for r in v),
373
388
  tps=median(r["gen_tps"] for r in v))
374
389
  t["spread"] = _spread(t["cells"])
@@ -404,7 +419,7 @@ def complete_models(rows, suite, test_ids=None):
404
419
  def compare_models(tests, models):
405
420
  """Per model over per_question entries that all `models` answered: passes, token and time
406
421
  totals/medians (totals add each question's median over repeats, so repeats don't inflate them), questions where it used the fewest tokens / was fastest (among passing
407
- answers for tokens; ties count for each), answers cut off."""
422
+ answers for tokens; ties count for each), answers cut off, mean score (partial credit)."""
408
423
  out = []
409
424
  for m in models:
410
425
  cs = [t["cells"][m] for t in tests]
@@ -421,6 +436,7 @@ def compare_models(tests, models):
421
436
  fastest += 1
422
437
  rs = [r for c in cs for r in c["rows"]]
423
438
  out.append({"model": m, "n": len(cs), "passed": passed, "total_tokens": total,
439
+ "score": statistics.fmean(c["score"] for c in cs) if cs else None,
424
440
  "avg_pp_tps": _rate([(r.get("prompt_tokens"), r["prompt_tps"]) for r in rs]),
425
441
  "avg_gen_tps": _rate([(r["tokens"], r["gen_tps"]) for r in rs]),
426
442
  "med_tokens": median(c["tokens"] for c in cs), "med_secs": median(c["secs"] for c in cs),
@@ -664,7 +680,8 @@ def main(argv):
664
680
  for note in settings_notes(infos):
665
681
  print("NOTE:", note)
666
682
  print_table(*scorecard(rows, rng))
667
- print("\nmean score = average score, with partial credit; all repeats = tests passed on every repeat.")
683
+ print("\nrepeats = answers per question (×1: one answer each, the roughest accuracy).")
684
+ print("mean score = average score, with partial credit; all repeats = tests passed on every repeat.")
668
685
  print("trunc = hit max_tokens; rep = output ends in a degenerate repeated pattern.")
669
686
  header, table = by_difficulty(rows)
670
687
  if any(r.get("difficulty") in LEVELS for r in rows):
tuieval/engine.py CHANGED
@@ -989,6 +989,14 @@ class Engine:
989
989
  need = {(t["id"], r) for t in pack.tests for r in range(self.certify_repeat(pack))}
990
990
  return "certified" if need <= keys else "screened"
991
991
 
992
+ def result_repeats(self, label, pack_name, tier="certify"):
993
+ """How many times each answered question of a stored result was asked: (fewest, most), or None."""
994
+ data = self._result(self.result_path(label, pack_name, tier))
995
+ if not data or not data["results"]:
996
+ return None
997
+ per_test = collections.Counter(r["test"] for r in data["results"])
998
+ return min(per_test.values()), max(per_test.values())
999
+
992
1000
  def _partial_lines(self, path):
993
1001
  """(header, answers) of an unfinished run's .partial.jsonl, cached like _result;
994
1002
  (None, []) if there is none."""
tuieval/insights.py CHANGED
@@ -6,8 +6,9 @@
6
6
  2 use cases accuracy per model and use case (pack group) as a heatmap; the best marked
7
7
  3 vs base a fine-tune against its base model (models.toml `base`): per pack, and the question
8
8
  categories it gained or lost most
9
- 4 why they fail each model's answers split into right, wrong, cut off, no answer / bad format, code
10
- crashed or hung, and critical failures
9
+ 4 why they fail each model's answers split into right, near miss (most checks passed), partly right,
10
+ wrong, cut off (or out of time), no answer / bad format, code crashed or hung, and
11
+ critical failures
11
12
  5 accuracy / cost accuracy against median tokens (or seconds) per question; the models no other beats
12
13
  on both
13
14
  6 questions questions every model fails (check them), every model passes (no signal), and the
@@ -267,7 +268,8 @@ def categories(rows, m, base, min_q=2):
267
268
 
268
269
 
269
270
  # ---------------------------------------------------------------- 4 why they fail
270
- KINDS = (("right", "green", "█"), ("wrong", "red", "█"), ("cut off", "yellow", "█"),
271
+ KINDS = (("right", "green", "█"), ("near miss", "#87af5f", "█"), ("partly right", "#af875f", "█"),
272
+ ("wrong", "red", "█"), ("cut off", "yellow", "█"),
271
273
  ("no answer / format", "magenta", "█"), ("code crashed or hung", "cyan", "█"), ("critical", "#ff5faf", "▓"))
272
274
  FORMAT = ("No 'ANSWER:' line", "Empty answer", "No code", "no tool call")
273
275
  CRASHED = ("Timed out after", "Harness crashed", "grader error")
@@ -278,13 +280,15 @@ def kind(r):
278
280
  return "right"
279
281
  if r.get("severity") == "critical":
280
282
  return "critical"
281
- if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET")):
283
+ if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET", "time limit")):
282
284
  return "cut off"
283
285
  if r["reason"].startswith(FORMAT):
284
286
  return "no answer / format"
285
287
  if r["reason"].startswith(CRASHED):
286
288
  return "code crashed or hung"
287
- return "wrong"
289
+ # partial credit (e.g. checks passed): close isn't the same as all wrong
290
+ score = r.get("score") or 0
291
+ return "near miss" if score >= compare.NEAR_MISS else "partly right" if score > 0 else "wrong"
288
292
 
289
293
 
290
294
  def _split(counts, total, width):
@@ -322,7 +326,9 @@ def failure_mix(rows, models, width=36):
322
326
  top, n = max(((k, c[k]) for k, *_ in KINDS[1:-1]), key=lambda x: x[1])
323
327
  if n / fails >= 0.4 and top != "wrong":
324
328
  actions.append(f"{m}: {n} of its {fails} failures are {top}: " + {
325
- "cut off": "it thinks too long for max_tokens, rather than reasoning wrong.",
329
+ "cut off": "it thinks too long for max_tokens (or the time limit), rather than reasoning wrong.",
330
+ "near miss": "they pass most of their checks: it's closer than its accuracy shows.",
331
+ "partly right": "they pass some of their checks, not none.",
326
332
  "no answer / format": "it doesn't follow the answer format (check its chat template and the "
327
333
  "pack's instructions).",
328
334
  "code crashed or hung": "its code crashes or never finishes."}[top])
tuieval/pta.py CHANGED
@@ -7,11 +7,17 @@
7
7
  model's ("28×").
8
8
  A accuracy the share of answers to those questions that passed (%).
9
9
 
10
+ Beside A, the score: partial credit averaged over the same answers (e.g. a code answer passing 5 of 6
11
+ checks scores 83%, where A counts it wrong). Shown when it differs from A; models can be ordered by it.
12
+
10
13
  The P and T bars are as long as the tokens and time used (the model that used the most fills its bar,
11
14
  the others in proportion, at least one block), so shorter is better; the A bar: longer is better.
12
15
  P and T also carry a 0-100 score (100 for the best, POINTS_PER_DOUBLING points less for every
13
16
  doubling), used to order models with the same accuracy.
14
17
 
18
+ Beside each name, the repeats: how many times it answered each of those questions ("×3", or "×1-3" when
19
+ it varies). A ×1 model's accuracy rests on one answer per question, so it is rougher than a ×3 one's.
20
+
15
21
  Models are compared on the questions all of them answered, so one that skipped some never looks
16
22
  faster. Server and connection errors aren't answers. By default a model with fewer than half as many
17
23
  answers as the most-answered model is left out (listed, so it can be picked), or it would shrink the
@@ -46,11 +52,15 @@ def memory_gb(infos):
46
52
  return out
47
53
 
48
54
 
49
- def index(rows, models=None, memory=None):
50
- """{"questions": n shared, "models": [{model, P, T, A, cut, cut_pct, tokens, total_s, passed, answers, alongside,
51
- fewest, fastest, tps, ttft, memory_gb}],
55
+ SORTS = ("accuracy", "score")
56
+
57
+
58
+ def index(rows, models=None, memory=None, sort="accuracy"):
59
+ """{"questions": n shared, "models": [{model, P, T, A, S, cut, cut_pct, tokens, total_s, passed, answers, alongside,
60
+ fewest, fastest, tps, ttft, memory_gb, repeats: (fewest, most) answers per question}],
52
61
  "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
53
- most-answered model}, best accuracy first. models: the ones to compare (default: every model in rows
62
+ most-answered model}, best accuracy (sort="accuracy") or score (sort="score") first. S: the mean
63
+ score (partial credit) in %. models: the ones to compare (default: every model in rows
54
64
  with enough answers; models named here are compared whatever their count). memory: memory_gb()."""
55
65
  no_answers = sorted({r["model"] for r in rows} - {r["model"] for r in rows if not r.get("error")})
56
66
  rows = [r for r in rows if not r.get("error")] # a server or connection error isn't an answer
@@ -75,6 +85,7 @@ def index(rows, models=None, memory=None):
75
85
  "total_s": sum(c["secs"] or 0 for c in cells),
76
86
  "passed": sum(c["passed"] for c in cells), "answers": answers,
77
87
  "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
88
+ "S": 100 * sum(c["score"] * c["n"] for c in cells) / answers if answers else None,
78
89
  # failed only because max_tokens ran out (overthinking, not wrong yet)
79
90
  "cut": sum(c["truncated"] for c in cells),
80
91
  "cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
@@ -84,6 +95,7 @@ def index(rows, models=None, memory=None):
84
95
  "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
85
96
  "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
86
97
  "memory_gb": (memory or {}).get(m),
98
+ "repeats": (min(c["n"] for c in cells), max(c["n"] for c in cells)) if cells else None,
87
99
  # answers that ran while other models were served shared the machine's speed
88
100
  "alongside": any(r.get("alongside") for c in cells for r in c["rows"])})
89
101
  fastest = min((x["total_s"] for x in out if x["total_s"] > 0), default=None)
@@ -95,7 +107,8 @@ def index(rows, models=None, memory=None):
95
107
  x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
96
108
  x["fewest"] = bool(fewest) and x["tokens"] == fewest
97
109
  x["fastest"] = bool(fastest) and x["total_s"] == fastest
98
- out.sort(key=lambda x: (-(x["A"] or 0), -(x["T"] or 0), x["model"]))
110
+ first, then = ("S", "A") if sort == "score" else ("A", "S")
111
+ out.sort(key=lambda x: (-(x[first] or 0), -(x[then] or 0), -(x["T"] or 0), x["model"]))
99
112
  return {"questions": len(shared), "models": out, "left_out": left_out, "no_answers": no_answers,
100
113
  "most": most}
101
114
 
@@ -107,6 +120,14 @@ LETTERS = (("P", "parsimony (tokens)", "▼ shorter is better", "tokens", "token
107
120
  ("A", "accuracy", "▲ longer is better", "A", "A", "green"))
108
121
 
109
122
 
123
+ def repeats(x):
124
+ """How many times a model answered each question compared: "×3", "×1-3", or "" when unknown."""
125
+ r = x.get("repeats")
126
+ if not r:
127
+ return ""
128
+ return f"×{r[0]}" if r[0] == r[1] else f"×{r[0]}-{r[1]}"
129
+
130
+
110
131
  def times(v):
111
132
  """A multiple of the best: "1.0×", "2.4×", "28×"."""
112
133
  return "-" if v is None else f"{v:.1f}×" if v < 9.95 else f"{v:.0f}×"
@@ -129,6 +150,8 @@ def bars(result, width=20, markup=True, click=None):
129
150
  if not result["models"]:
130
151
  return []
131
152
  name_w = max(len(x["model"]) for x in result["models"])
153
+ rep_w = max(len(repeats(x)) for x in result["models"])
154
+ name_w += rep_w + 1 if rep_w else 0 # the repeats sit after the name, in its column
132
155
  # each column: the bar, a space, "100%" or "9.8×" (6), then a gap; at least as wide as its heading
133
156
  cell_w = max(width + 9, *(len(t) + 2 for k, name, better, *_ in LETTERS for t in (f"{k} {name}", better)))
134
157
  pad = " " * (name_w + 2)
@@ -145,6 +168,10 @@ def bars(result, width=20, markup=True, click=None):
145
168
  if v is not None and v < 100: # full only when every answer passed
146
169
  full = min(full, width - 1)
147
170
  num = "-" if v is None else compare.pct(v / 100)
171
+ score = x.get("S")
172
+ if score is not None and v is not None and compare.pct(score / 100) != num:
173
+ note = f"score {compare.pct(score / 100)}"
174
+ num = f"{num:>6} " + (f"[dim]{note}[/dim]" if markup else note)
148
175
  cut = x.get("cut_pct") or 0
149
176
  part = min(width - full, max(1, round(width * cut / 100))) if cut else 0
150
177
  if cut: # after the accuracy, which stays aligned with the other rows'
@@ -160,11 +187,14 @@ def bars(result, width=20, markup=True, click=None):
160
187
  bar = (f"[{color}]{bar}[/]" + (f"[#ff8c00]{'█' * part}[/]" if part else "") + f"[grey37]{rest}[/]"
161
188
  if markup else bar + cut_bar + rest)
162
189
  cells.append(f"{bar} {num:>6}" + " " * max(0, cell_w - width - 1 - max(6, len(num))))
163
- name = f"{x['model']:<{name_w}}"
190
+ tag = repeats(x)
191
+ name = f"{x['model'] + (' ' + tag if tag else ''):<{name_w}}"
164
192
  if markup:
165
- pad_w = name_w - len(x["model"])
166
- name = (f"[@click={click}({row})]{rich_escape(x['model'])}[/]" + " " * pad_w if click
167
- else rich_escape(name))
193
+ pad_w = name_w - len(x["model"]) - (len(tag) + 1 if tag else 0)
194
+ # a single answer per question in yellow: its accuracy is the roughest
195
+ tag = f" [{'yellow' if x['repeats'][1] == 1 else 'dim'}]{tag}[/]" if tag else ""
196
+ name = ((f"[@click={click}({row})]{rich_escape(x['model'])}[/]" if click else rich_escape(x["model"]))
197
+ + tag + " " * pad_w)
168
198
  lines.append((f"{name} " + "".join(cells)).rstrip())
169
199
  return lines
170
200
 
@@ -189,11 +219,12 @@ def table(result):
189
219
  """(header, rows) for a plain table of the index."""
190
220
  n = result["questions"] or 1
191
221
  over = any(x.get("over") for x in result["models"]) # a column only when some answer ran past max_tokens
192
- header = ["model", "tokens/answer", "time/answer", "answers right", "truncated", *(["over budget"] if over else []),
222
+ header = ["model", "repeats", "tokens/answer", "time/answer", "answers right", "score", "truncated", *(["over budget"] if over else []),
193
223
  "tok/s", "TTFT s", "memory GB"]
194
- rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
224
+ rows = [[x["model"], repeats(x) or "-", f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
195
225
  fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
196
- f"{x['passed']}/{x['answers']}", f"{x.get('cut') or 0}/{x['answers']}",
226
+ f"{x['passed']}/{x['answers']}", "-" if x.get("S") is None else compare.pct(x["S"] / 100),
227
+ f"{x.get('cut') or 0}/{x['answers']}",
197
228
  *([f"{x.get('over') or 0}/{x['answers']}"] if over else []),
198
229
  _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
199
230
  _num(x.get("memory_gb"), ".1f")] for x in result["models"]]
@@ -228,7 +259,11 @@ def scope_note(result, models):
228
259
  return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
229
260
  "P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
230
261
  "and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
231
- "(longer is better)"
262
+ "(longer is better); score: partial credit, e.g. 5 of 6 checks passed is 83% (shown when it differs)"
263
+ + (f"; ×n beside a name: answers per question. ×1 models answered each question once, so one "
264
+ f"answer more or less right moves their accuracy by {100 / result['questions']:.0f} points; "
265
+ "repeats steady it" if len({x.get("repeats") for x in result["models"]}) > 1
266
+ or any(x.get("repeats") and x["repeats"][1] == 1 for x in result["models"]) else "")
232
267
  + ("; orange, (n% truncated): answers cut off at max_tokens. They count as wrong, but the model ran "
233
268
  "out of tokens rather than answering wrong." if any(x.get("cut") for x in result["models"]) else ".")
234
269
  + (" Over budget: answers that used more tokens than the max_tokens sent, because the provider didn't "
@@ -236,14 +271,14 @@ def scope_note(result, models):
236
271
  if any(x.get("over") for x in result["models"]) else ""))
237
272
 
238
273
 
239
- def for_engine(e, labels=None, packs=None, results_dir=None):
274
+ def for_engine(e, labels=None, packs=None, results_dir=None, sort="accuracy"):
240
275
  """(result, models) from a workspace's finished results (results on their way out excluded)."""
241
276
  paths = [p for p in compare.default_paths(results_dir or e.results_dir) if not e.superseded(p)]
242
277
  rows, infos = compare.load(paths) if paths else ([], [])
243
278
  if packs:
244
279
  rows = [r for r in rows if r["suite"] in packs]
245
280
  models = sorted({r["model"] for r in rows} & set(labels) if labels else {r["model"] for r in rows})
246
- return index(rows, models if labels else None, memory_gb(infos)), models
281
+ return index(rows, models if labels else None, memory_gb(infos), sort), models
247
282
 
248
283
 
249
284
  def markdown(result, models):
tuieval/run_evals.py CHANGED
@@ -402,6 +402,8 @@ def cmd_pta(argv):
402
402
  p.add_argument("--only", help="comma-separated model labels (default: every model with results)")
403
403
  p.add_argument("--packs", help="comma-separated packs (default: all)")
404
404
  p.add_argument("--width", type=int, default=20, help="bar width in characters (default 20)")
405
+ p.add_argument("--sort", choices=("accuracy", "score"), default="accuracy",
406
+ help="order models by answers right (default) or by score (partial credit)")
405
407
  p.add_argument("--models")
406
408
  p.add_argument("--results-dir")
407
409
  a = p.parse_args(argv)
@@ -410,7 +412,7 @@ def cmd_pta(argv):
410
412
  e = engine.Engine(models_path=a.models, results_dir=a.results_dir)
411
413
  labels = [x.strip() for x in a.only.split(",")] if a.only else None
412
414
  packs = [x.strip() for x in a.packs.split(",")] if a.packs else None
413
- result, models = pta.for_engine(e, labels, packs)
415
+ result, models = pta.for_engine(e, labels, packs, sort=a.sort)
414
416
  out = Console(highlight=False)
415
417
  if not models:
416
418
  sys.exit("no finished results to compare" + (" for those models or packs" if labels or packs else ""))
tuieval/tui.py CHANGED
@@ -937,7 +937,10 @@ class SetupScreen(Screen):
937
937
  for n in e.packs:
938
938
  st = e.result_status(label, n)
939
939
  if st:
940
- packs.append(f"{mark.get(st, '[yellow]~[/yellow]')}{n}")
940
+ reps = e.result_repeats(label, n) if st in ("certified", "screened") else None
941
+ times = ("" if not reps else f"[dim]×{reps[0]}[/dim]" if reps[0] == reps[1]
942
+ else f"[dim]×{reps[0]}-{reps[1]}[/dim]")
943
+ packs.append(f"{mark.get(st, '[yellow]~[/yellow]')}{n}{times}")
941
944
  if st.startswith("outdated"):
942
945
  why.setdefault(st.removeprefix("outdated: "), []).append(n)
943
946
  if why: # each reason once, not once per pack
@@ -951,7 +954,7 @@ class SetupScreen(Screen):
951
954
  hidden = (f" [cyan]{' '.join(tags)}[/cyan]" if tags else "") + hidden
952
955
  detail.update(f"[b]{label}[/b] [dim]{m['server']} · {m.get('model') or ''}[/dim]"
953
956
  f"{serving_badge(e, m, not e.has_results())}{hidden}\n{verdicts}\n"
954
- + ("[dim]results (✓ certified ◐ screened ~ outdated … partial):[/dim] " + " ".join(packs)
957
+ + ("[dim]results (✓ certified ◐ screened ~ outdated … partial; ×n answers per question):[/dim] " + " ".join(packs)
955
958
  if packs else "[dim]no results yet[/dim]"))
956
959
 
957
960
  @on(Checkbox.Changed, "#show-hidden")
@@ -2892,6 +2895,10 @@ class ResultsScreen(Screen):
2892
2895
  def action_runs(self):
2893
2896
  self.app.open_sessions()
2894
2897
 
2898
+ FAIL_KINDS = [("Every kind", ""), ("Near misses", "near miss"), ("Partly right", "partly right"),
2899
+ ("Wrong", "wrong"), ("Cut off", "cut off"), ("No answer / format", "no answer / format"),
2900
+ ("Code crashed or hung", "code crashed or hung"), ("Critical", "critical")]
2901
+
2895
2902
  READINESS_HELP = ("[dim]PASS needs a Certify run with zero critical failures over enough trials and an "
2896
2903
  "accuracy lower bound that clears the pack's gate (pack.toml). Screening can only FAIL "
2897
2904
  "or look promising.[/dim]")
@@ -2913,6 +2920,8 @@ class ResultsScreen(Screen):
2913
2920
  with TabPane("PTA index", id="tab-pta"):
2914
2921
  with Horizontal(id="pta-controls"):
2915
2922
  yield Select([("All packs", "")], id="pta-pack", allow_blank=False, value="")
2923
+ yield Select([("By accuracy", "accuracy"), ("By score", "score")], id="pta-sort",
2924
+ allow_blank=False, value="accuracy")
2916
2925
  yield Button("Models: all", id="pta-models")
2917
2926
  yield Static("[dim]P parsimony (tokens): tokens used, reasoning and answer. T time: total time. "
2918
2927
  "A accuracy: answers right. Verdicts and critical failures are in Production "
@@ -2954,6 +2963,7 @@ class ResultsScreen(Screen):
2954
2963
  with TabPane("Failures", id="tab-failures"):
2955
2964
  with Horizontal(id="fail-controls"):
2956
2965
  yield Button("Models: all", id="fail-models")
2966
+ yield Select(self.FAIL_KINDS, id="fail-kind", allow_blank=False, value="")
2957
2967
  yield Static(id="fail-count")
2958
2968
  yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
2959
2969
  yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
@@ -3045,24 +3055,40 @@ class ResultsScreen(Screen):
3045
3055
  self.sort_tables()
3046
3056
 
3047
3057
  def fill_failures(self):
3048
- """Every wrong answer, of the models picked (Models: shared with the other tabs)."""
3058
+ """Every wrong answer, of the models picked (Models: shared with the other tabs), of the kind picked
3059
+ (insights.kind: near miss, partly right, wrong, cut off, …)."""
3060
+ from . import insights
3049
3061
  f = self.query_one("#failures", DataTable)
3050
3062
  f.clear(columns=True)
3051
- f.add_columns("Model", "Level", "Test", "Answered", "Reason")
3063
+ f.add_columns("Model", "Level", "Test", "Answered", "Kind", "Reason")
3052
3064
  every = {r["model"] for r in self.rows}
3053
3065
  chosen = {m for m in self.app.pq_models if m in every}
3054
- rows = [r for r in self.rows if not chosen or r["model"] in chosen]
3066
+ wanted = self.query_one("#fail-kind", Select).value or ""
3067
+ rows = [r for r in self.rows if (not chosen or r["model"] in chosen)
3068
+ and (r["ok"] or not wanted or insights.kind(r) == wanted)]
3055
3069
  # compare.failures lists the failed rows in order, so they line up with fail_rows
3056
3070
  self.fail_rows = [r for r in rows if not r["ok"]]
3071
+ colors = {k: c for k, c, _ in insights.KINDS}
3057
3072
  for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
3058
- when = dates.day(dates.answered_at(self.fail_rows[i].get("raw") or {}))
3059
- f.add_row(model, diff_badge(level), rich_escape(test), when, rich_escape(reason[:130]), key=str(i))
3073
+ r = self.fail_rows[i]
3074
+ when = dates.day(dates.answered_at(r.get("raw") or {}))
3075
+ k = insights.kind(r)
3076
+ checks = compare.checks_passed(r)
3077
+ label = f"{k} {checks[0]}/{checks[1]}" if checks and k in ("near miss", "partly right", "wrong") else k
3078
+ f.add_row(model, diff_badge(level), rich_escape(test), when, f"[{colors[k]}]{label}[/]",
3079
+ rich_escape(reason[:130]), key=str(i))
3060
3080
  self.query_one("#fail-models", Button).label = self.models_label(chosen)
3061
3081
  n = len(self.fail_rows)
3062
3082
  self.query_one("#fail-count", Static).update(
3063
- f"{n} failure{'s' * (n != 1)}" + (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
3083
+ f"{n} {'' if not wanted else wanted + ' '}failure{'s' * (n != 1)}"
3084
+ + (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
3064
3085
  self.sort_table(f)
3065
3086
 
3087
+ @on(Select.Changed, "#fail-kind")
3088
+ def fail_kind_changed(self):
3089
+ if getattr(self, "rows", None) is not None:
3090
+ self.fill_failures()
3091
+
3066
3092
  @on(DataTable.RowSelected, "#failures")
3067
3093
  def failure_selected(self, event):
3068
3094
  # in the order shown, so [ and ] step through them as sorted
@@ -3223,6 +3249,13 @@ class ResultsScreen(Screen):
3223
3249
  out = insights.panels(rows, models, e.packs, ref, bases, by, engine=e if main else None)
3224
3250
  body.update("\n\n".join(insights.render(p, i) for i, p in enumerate(out, 1)))
3225
3251
 
3252
+ @on(Select.Changed, "#pta-sort")
3253
+ def pta_sort_changed(self):
3254
+ """The Sort dropdown orders the bars and the table, taking over from a column sorted by a click."""
3255
+ self.app.result_sorts.pop("pta", None)
3256
+ if getattr(self, "rows", None) is not None:
3257
+ self.fill_pta()
3258
+
3226
3259
  @on(Select.Changed, "#pta-pack")
3227
3260
  @on(Select.Changed, "#pq-pack")
3228
3261
  def pack_changed(self, event):
@@ -3293,7 +3326,8 @@ class ResultsScreen(Screen):
3293
3326
  self.query_one("#pta-left-out", Static).update("")
3294
3327
  return
3295
3328
  # none picked: every model with enough answers (pta.index leaves out the rest, and says so)
3296
- result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos))
3329
+ result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos),
3330
+ self.query_one("#pta-sort", Select).value or "accuracy")
3297
3331
  self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
3298
3332
  self.pta_result = result
3299
3333
  removed = [m for m in self.pta_removed_now() if m in present]
@@ -3439,7 +3473,8 @@ class ResultsScreen(Screen):
3439
3473
  tt = r["total_secs"]
3440
3474
  cells.append([r["model"],
3441
3475
  hi(tt, ("-" if not tt else f"{tt:.1f}s" if tt < 60 else fmt_secs(tt)), "total_secs", "min"),
3442
- hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max"),
3476
+ hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max")
3477
+ + (f" [dim]{r['score']:.0%}[/dim]" if r["passed"] < r["n"] and r["score"] else ""),
3443
3478
  hi(r["med_tokens"], f"{r['med_tokens']:,.0f}" if r["med_tokens"] else "-", "med_tokens", "min"),
3444
3479
  hi(r["total_tokens"], f"{r['total_tokens']:,.0f}", "total_tokens", "min"),
3445
3480
  hi(r["tokens_per_pass"], f"{r['tokens_per_pass']:,.0f}" if r["tokens_per_pass"] else "-",
@@ -3469,7 +3504,7 @@ class ResultsScreen(Screen):
3469
3504
  "shared the machine's speed: compare their times with care.[/yellow]")
3470
3505
  title.update(line + " [dim]Time / Tokens: each question's median, added up. Med tok / Med s: median "
3471
3506
  "per answer. Tok/✓: tokens per answer right. Prompt t/s / Gen t/s: all tokens over all the time "
3472
- "they took (cached prompts left out). Fewest / Fastest: questions won; ties count for both. "
3507
+ "they took (cached prompts left out). Passed: questions right on every repeat, then the score (partial credit, e.g. checks passed) when some failed. Fewest / Fastest: questions won; ties count for both. "
3473
3508
  "Cut off: answers that hit max_tokens.[/dim]")
3474
3509
 
3475
3510
  @staticmethod
@@ -3478,6 +3513,9 @@ class ResultsScreen(Screen):
3478
3513
  else f"[yellow]{c['passed']}/{c['n']}[/yellow]")
3479
3514
  if c["n"] > 1 and c["passed"] in (0, c["n"]):
3480
3515
  mark += f"[dim]{c['n']}[/dim]"
3516
+ if c["passed"] < c["n"] and c.get("score"): # partial credit: how close it came
3517
+ checks = compare.checks_passed(c["rows"][0]) if c["n"] == 1 else None
3518
+ mark += f" [yellow]{checks[0]}/{checks[1]}[/yellow]" if checks else f" [yellow]{c['score']:.0%}[/yellow]"
3481
3519
  tok = c["tokens"]
3482
3520
  tok_s = "-" if not tok else (f"{tok / 1000:.1f}k" if tok >= 1000 else f"{tok:.0f}")
3483
3521
  if tok and best and tok == best and c["passed"]:
@@ -3624,6 +3662,7 @@ class EvalsApp(App):
3624
3662
  #ins-controls #ins-cost { width: 20; }
3625
3663
  #ins-body { height: 1fr; }
3626
3664
  #pta-controls Select { width: 28; margin-right: 1; }
3665
+ #pta-controls #pta-sort { width: 18; }
3627
3666
  #pta-bars { height: auto; padding: 1 0 0 0; }
3628
3667
  #pta { width: 1fr; height: auto; margin-top: 1; }
3629
3668
  #pta-left-out { height: auto; padding: 1 0 0 0; }
@@ -3632,6 +3671,7 @@ class EvalsApp(App):
3632
3671
  #pq-models { margin-right: 1; min-width: 20; }
3633
3672
  #fail-controls { height: 3; }
3634
3673
  #fail-models { margin-right: 1; min-width: 20; }
3674
+ #fail-kind { width: 28; margin-right: 1; }
3635
3675
  #fail-count { width: auto; padding: 1 1; }
3636
3676
  #pq-compare-title { padding: 0 1; height: auto; }
3637
3677
  #pq-summary { height: auto; } /* every picked model, no scrolling: the tab scrolls instead */
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: tuieval
3
- Version: 0.2.0.dev129
3
+ Version: 0.2.0.dev133
4
4
  Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
5
5
  Project-URL: Homepage, https://github.com/ashe-wb/tuieval
6
6
  Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -1,27 +1,27 @@
1
1
  tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
2
2
  tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
3
- tuieval/_version.py,sha256=AkC3QCfZjjCHBABegFa0MlKWs5cF5ey3LXR1zHC6wgU,537
3
+ tuieval/_version.py,sha256=seXDvsrsCa7COI9ADByO7-ByiyH-SLyoBwimkvhuD0c,537
4
4
  tuieval/agents.py,sha256=XmY4d4PAnC7BQcs_6KO9WTuIdpnwYVAdyhqItybWdAM,15408
5
5
  tuieval/builtin.toml,sha256=romPxP9o3r_4z3V8bdsmE9oUk0G2plDeqygPbqrwOys,4529
6
6
  tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
7
7
  tuieval/client.py,sha256=5wo0ySMD3o8up3cxKZicUWYFvOLZB60kYBV7yxP6LjI,12378
8
- tuieval/compare.py,sha256=WWnkGTm2F0-hhUG8EU-1TmhDdg3e3sFXo57kyU3dzck,36242
8
+ tuieval/compare.py,sha256=L5mmNY7ZUSOVG9p-_2vSZ9227TVguvIm6ZZ0n56vds8,37280
9
9
  tuieval/dates.py,sha256=CVazlwNjCRhD__D12zXLdeqF2ySt15X3xLUQeE-o6PY,6293
10
10
  tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
11
- tuieval/engine.py,sha256=M78wy43ZL4qmwUaZBDhpRZH5dWQKd-o0z59kWg7waCQ,136050
11
+ tuieval/engine.py,sha256=lbdESO0J1kiZUCJ4Rl6s54wNXGJq5LuDwgm7sKR-fGc,136497
12
12
  tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
13
- tuieval/insights.py,sha256=ZIINv-iTFOomFK29pXF7xY_w-mf-_d4T1da4NuiJqiQ,28964
13
+ tuieval/insights.py,sha256=BDOn9UkDUkCKFcG2YIfZcnECgIFIUHxQawPRROGvDpM,29534
14
14
  tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
15
15
  tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
16
16
  tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
17
17
  tuieval/packs.py,sha256=KOj0BkwsB2124aslGyauZonyyGqtSrCJ-WAozEntmqE,17466
18
18
  tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
19
- tuieval/pta.py,sha256=BZbstnZFWpYmCdj9I0dbjXvLvhFhDCfB5vrRNWsAC2A,14493
19
+ tuieval/pta.py,sha256=K8KE_PUcOgoZGaJHCWkPYfVHbPvd243uG0QPuxnVKto,16949
20
20
  tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
21
- tuieval/run_evals.py,sha256=rbe1ae4gDUauoQMubfxLHfbokgN2OT683siuksdSeFo,50490
21
+ tuieval/run_evals.py,sha256=afZIzSdZPSCWnhc9qV_L1TXexl7YgSF6eKlV7RimMao,50679
22
22
  tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
23
23
  tuieval/selftest.py,sha256=cIGSDCrCK-QISAXjKSJDB4sD-WwS32bn3Gn4Y0oGoF8,11502
24
- tuieval/tui.py,sha256=9ckEs-Hvu5nqUi9FNQGB0ydiDJyoQVU8TeoMqxu05_Q,201891
24
+ tuieval/tui.py,sha256=-7eJASdO9OhFz2KJoXNKtqeTL9DlsyafZa3mPI-tjfs,204533
25
25
  tuieval/tune.py,sha256=izz1Qs-d9PrWAqEZpyrLxfrlhlaouif9uFQSeJveL0k,41294
26
26
  tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
27
27
  tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -50,8 +50,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
50
50
  tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
51
51
  tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
52
52
  tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
53
- tuieval-0.2.0.dev129.dist-info/METADATA,sha256=66u5WjFPf7oURTEeNM8GVisRgnNh8ohFCEyg08G7AXc,15105
54
- tuieval-0.2.0.dev129.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
55
- tuieval-0.2.0.dev129.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
56
- tuieval-0.2.0.dev129.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
57
- tuieval-0.2.0.dev129.dist-info/RECORD,,
53
+ tuieval-0.2.0.dev133.dist-info/METADATA,sha256=dLL6ycED3j35icT3OH8DRUbgMfLBpIUXtQPwMz_z7dI,15105
54
+ tuieval-0.2.0.dev133.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
55
+ tuieval-0.2.0.dev133.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
56
+ tuieval-0.2.0.dev133.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
57
+ tuieval-0.2.0.dev133.dist-info/RECORD,,