tuieval 0.2.0.dev129__py3-none-any.whl → 0.2.0.dev131__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tuieval/_version.py CHANGED
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
18
18
  commit_id: str | None
19
19
  __commit_id__: str | None
20
20
 
21
- __version__ = version = '0.2.0.dev129'
22
- __version_tuple__ = version_tuple = (0, 2, 0, 'dev129')
21
+ __version__ = version = '0.2.0.dev131'
22
+ __version_tuple__ = version_tuple = (0, 2, 0, 'dev131')
23
23
 
24
24
  __commit_id__ = commit_id = None
tuieval/compare.py CHANGED
@@ -351,10 +351,22 @@ def speed(rows, infos, machine=None, speeds=None):
351
351
  return header, table, frontier
352
352
 
353
353
 
354
+ NEAR_MISS = 0.8 # a failed answer scoring at least this (e.g. 5 of 6 checks) is a near miss
355
+
356
+
357
+ def checks_passed(r):
358
+ """(passed, total) checks of a failed code answer (from its stored checks, or its reason), else None."""
359
+ checks = (r.get("raw") or {}).get("checks")
360
+ if checks:
361
+ return sum(v is None for v in checks.values()), len(checks)
362
+ m = re.match(r"Failed (\d+)/(\d+) checks", r.get("reason") or "")
363
+ return (int(m[2]) - int(m[1]), int(m[2])) if m else None
364
+
365
+
354
366
  def per_question(rows):
355
367
  """[{test, name, suite, difficulty, cells: {model: cell}, spread}] with one cell per model that
356
- answered: n answers, passes, median tokens / seconds / tok/s over repeats, truncated count,
357
- partial (some answers from a pack still running), rows (the answers, for the detail view).
368
+ answered: n answers, passes, mean score (partial credit), median tokens / seconds / tok/s over repeats,
369
+ truncated count, partial (some answers from a pack still running), rows (the answers, for the detail view).
358
370
  spread = {"tokens": .., "secs": ..}: largest / smallest model median (1 if one model)."""
359
371
  tests = {}
360
372
  for r in rows:
@@ -369,6 +381,7 @@ def per_question(rows):
369
381
  c.update(rows=v, n=len(v), passed=sum(r["ok"] for r in v), truncated=sum(r["truncated"] for r in v),
370
382
  over=sum(bool(r.get("over_budget")) for r in v),
371
383
  partial=any(r.get("partial") for r in v),
384
+ score=statistics.fmean(r.get("score", r["ok"]) for r in v),
372
385
  tokens=median(r["tokens"] or None for r in v), secs=median(r["latency"] / 1000 or None for r in v),
373
386
  tps=median(r["gen_tps"] for r in v))
374
387
  t["spread"] = _spread(t["cells"])
@@ -404,7 +417,7 @@ def complete_models(rows, suite, test_ids=None):
404
417
  def compare_models(tests, models):
405
418
  """Per model over per_question entries that all `models` answered: passes, token and time
406
419
  totals/medians (totals add each question's median over repeats, so repeats don't inflate them), questions where it used the fewest tokens / was fastest (among passing
407
- answers for tokens; ties count for each), answers cut off."""
420
+ answers for tokens; ties count for each), answers cut off, mean score (partial credit)."""
408
421
  out = []
409
422
  for m in models:
410
423
  cs = [t["cells"][m] for t in tests]
@@ -421,6 +434,7 @@ def compare_models(tests, models):
421
434
  fastest += 1
422
435
  rs = [r for c in cs for r in c["rows"]]
423
436
  out.append({"model": m, "n": len(cs), "passed": passed, "total_tokens": total,
437
+ "score": statistics.fmean(c["score"] for c in cs) if cs else None,
424
438
  "avg_pp_tps": _rate([(r.get("prompt_tokens"), r["prompt_tps"]) for r in rs]),
425
439
  "avg_gen_tps": _rate([(r["tokens"], r["gen_tps"]) for r in rs]),
426
440
  "med_tokens": median(c["tokens"] for c in cs), "med_secs": median(c["secs"] for c in cs),
tuieval/insights.py CHANGED
@@ -6,8 +6,9 @@
6
6
  2 use cases accuracy per model and use case (pack group) as a heatmap; the best marked
7
7
  3 vs base a fine-tune against its base model (models.toml `base`): per pack, and the question
8
8
  categories it gained or lost most
9
- 4 why they fail each model's answers split into right, wrong, cut off, no answer / bad format, code
10
- crashed or hung, and critical failures
9
+ 4 why they fail each model's answers split into right, near miss (most checks passed), partly right,
10
+ wrong, cut off (or out of time), no answer / bad format, code crashed or hung, and
11
+ critical failures
11
12
  5 accuracy / cost accuracy against median tokens (or seconds) per question; the models no other beats
12
13
  on both
13
14
  6 questions questions every model fails (check them), every model passes (no signal), and the
@@ -267,7 +268,8 @@ def categories(rows, m, base, min_q=2):
267
268
 
268
269
 
269
270
  # ---------------------------------------------------------------- 4 why they fail
270
- KINDS = (("right", "green", "█"), ("wrong", "red", "█"), ("cut off", "yellow", "█"),
271
+ KINDS = (("right", "green", "█"), ("near miss", "#87af5f", "█"), ("partly right", "#af875f", "█"),
272
+ ("wrong", "red", "█"), ("cut off", "yellow", "█"),
271
273
  ("no answer / format", "magenta", "█"), ("code crashed or hung", "cyan", "█"), ("critical", "#ff5faf", "▓"))
272
274
  FORMAT = ("No 'ANSWER:' line", "Empty answer", "No code", "no tool call")
273
275
  CRASHED = ("Timed out after", "Harness crashed", "grader error")
@@ -278,13 +280,15 @@ def kind(r):
278
280
  return "right"
279
281
  if r.get("severity") == "critical":
280
282
  return "critical"
281
- if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET")):
283
+ if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET", "time limit")):
282
284
  return "cut off"
283
285
  if r["reason"].startswith(FORMAT):
284
286
  return "no answer / format"
285
287
  if r["reason"].startswith(CRASHED):
286
288
  return "code crashed or hung"
287
- return "wrong"
289
+ # partial credit (e.g. checks passed): close isn't the same as all wrong
290
+ score = r.get("score") or 0
291
+ return "near miss" if score >= compare.NEAR_MISS else "partly right" if score > 0 else "wrong"
288
292
 
289
293
 
290
294
  def _split(counts, total, width):
@@ -322,7 +326,9 @@ def failure_mix(rows, models, width=36):
322
326
  top, n = max(((k, c[k]) for k, *_ in KINDS[1:-1]), key=lambda x: x[1])
323
327
  if n / fails >= 0.4 and top != "wrong":
324
328
  actions.append(f"{m}: {n} of its {fails} failures are {top}: " + {
325
- "cut off": "it thinks too long for max_tokens, rather than reasoning wrong.",
329
+ "cut off": "it thinks too long for max_tokens (or the time limit), rather than reasoning wrong.",
330
+ "near miss": "they pass most of their checks: it's closer than its accuracy shows.",
331
+ "partly right": "they pass some of their checks, not none.",
326
332
  "no answer / format": "it doesn't follow the answer format (check its chat template and the "
327
333
  "pack's instructions).",
328
334
  "code crashed or hung": "its code crashes or never finishes."}[top])
tuieval/pta.py CHANGED
@@ -7,6 +7,9 @@
7
7
  model's ("28×").
8
8
  A accuracy the share of answers to those questions that passed (%).
9
9
 
10
+ Beside A, the score: partial credit averaged over the same answers (e.g. a code answer passing 5 of 6
11
+ checks scores 83%, where A counts it wrong). Shown when it differs from A; models can be ordered by it.
12
+
10
13
  The P and T bars are as long as the tokens and time used (the model that used the most fills its bar,
11
14
  the others in proportion, at least one block), so shorter is better; the A bar: longer is better.
12
15
  P and T also carry a 0-100 score (100 for the best, POINTS_PER_DOUBLING points less for every
@@ -46,11 +49,15 @@ def memory_gb(infos):
46
49
  return out
47
50
 
48
51
 
49
- def index(rows, models=None, memory=None):
50
- """{"questions": n shared, "models": [{model, P, T, A, cut, cut_pct, tokens, total_s, passed, answers, alongside,
52
+ SORTS = ("accuracy", "score")
53
+
54
+
55
+ def index(rows, models=None, memory=None, sort="accuracy"):
56
+ """{"questions": n shared, "models": [{model, P, T, A, S, cut, cut_pct, tokens, total_s, passed, answers, alongside,
51
57
  fewest, fastest, tps, ttft, memory_gb}],
52
58
  "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
53
- most-answered model}, best accuracy first. models: the ones to compare (default: every model in rows
59
+ most-answered model}, best accuracy (sort="accuracy") or score (sort="score") first. S: the mean
60
+ score (partial credit) in %. models: the ones to compare (default: every model in rows
54
61
  with enough answers; models named here are compared whatever their count). memory: memory_gb()."""
55
62
  no_answers = sorted({r["model"] for r in rows} - {r["model"] for r in rows if not r.get("error")})
56
63
  rows = [r for r in rows if not r.get("error")] # a server or connection error isn't an answer
@@ -75,6 +82,7 @@ def index(rows, models=None, memory=None):
75
82
  "total_s": sum(c["secs"] or 0 for c in cells),
76
83
  "passed": sum(c["passed"] for c in cells), "answers": answers,
77
84
  "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
85
+ "S": 100 * sum(c["score"] * c["n"] for c in cells) / answers if answers else None,
78
86
  # failed only because max_tokens ran out (overthinking, not wrong yet)
79
87
  "cut": sum(c["truncated"] for c in cells),
80
88
  "cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
@@ -95,7 +103,8 @@ def index(rows, models=None, memory=None):
95
103
  x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
96
104
  x["fewest"] = bool(fewest) and x["tokens"] == fewest
97
105
  x["fastest"] = bool(fastest) and x["total_s"] == fastest
98
- out.sort(key=lambda x: (-(x["A"] or 0), -(x["T"] or 0), x["model"]))
106
+ first, then = ("S", "A") if sort == "score" else ("A", "S")
107
+ out.sort(key=lambda x: (-(x[first] or 0), -(x[then] or 0), -(x["T"] or 0), x["model"]))
99
108
  return {"questions": len(shared), "models": out, "left_out": left_out, "no_answers": no_answers,
100
109
  "most": most}
101
110
 
@@ -145,6 +154,10 @@ def bars(result, width=20, markup=True, click=None):
145
154
  if v is not None and v < 100: # full only when every answer passed
146
155
  full = min(full, width - 1)
147
156
  num = "-" if v is None else compare.pct(v / 100)
157
+ score = x.get("S")
158
+ if score is not None and v is not None and compare.pct(score / 100) != num:
159
+ note = f"score {compare.pct(score / 100)}"
160
+ num = f"{num:>6} " + (f"[dim]{note}[/dim]" if markup else note)
148
161
  cut = x.get("cut_pct") or 0
149
162
  part = min(width - full, max(1, round(width * cut / 100))) if cut else 0
150
163
  if cut: # after the accuracy, which stays aligned with the other rows'
@@ -189,11 +202,12 @@ def table(result):
189
202
  """(header, rows) for a plain table of the index."""
190
203
  n = result["questions"] or 1
191
204
  over = any(x.get("over") for x in result["models"]) # a column only when some answer ran past max_tokens
192
- header = ["model", "tokens/answer", "time/answer", "answers right", "truncated", *(["over budget"] if over else []),
205
+ header = ["model", "tokens/answer", "time/answer", "answers right", "score", "truncated", *(["over budget"] if over else []),
193
206
  "tok/s", "TTFT s", "memory GB"]
194
207
  rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
195
208
  fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
196
- f"{x['passed']}/{x['answers']}", f"{x.get('cut') or 0}/{x['answers']}",
209
+ f"{x['passed']}/{x['answers']}", "-" if x.get("S") is None else compare.pct(x["S"] / 100),
210
+ f"{x.get('cut') or 0}/{x['answers']}",
197
211
  *([f"{x.get('over') or 0}/{x['answers']}"] if over else []),
198
212
  _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
199
213
  _num(x.get("memory_gb"), ".1f")] for x in result["models"]]
@@ -228,7 +242,7 @@ def scope_note(result, models):
228
242
  return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
229
243
  "P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
230
244
  "and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
231
- "(longer is better)"
245
+ "(longer is better); score: partial credit, e.g. 5 of 6 checks passed is 83% (shown when it differs)"
232
246
  + ("; orange, (n% truncated): answers cut off at max_tokens. They count as wrong, but the model ran "
233
247
  "out of tokens rather than answering wrong." if any(x.get("cut") for x in result["models"]) else ".")
234
248
  + (" Over budget: answers that used more tokens than the max_tokens sent, because the provider didn't "
@@ -236,14 +250,14 @@ def scope_note(result, models):
236
250
  if any(x.get("over") for x in result["models"]) else ""))
237
251
 
238
252
 
239
- def for_engine(e, labels=None, packs=None, results_dir=None):
253
+ def for_engine(e, labels=None, packs=None, results_dir=None, sort="accuracy"):
240
254
  """(result, models) from a workspace's finished results (results on their way out excluded)."""
241
255
  paths = [p for p in compare.default_paths(results_dir or e.results_dir) if not e.superseded(p)]
242
256
  rows, infos = compare.load(paths) if paths else ([], [])
243
257
  if packs:
244
258
  rows = [r for r in rows if r["suite"] in packs]
245
259
  models = sorted({r["model"] for r in rows} & set(labels) if labels else {r["model"] for r in rows})
246
- return index(rows, models if labels else None, memory_gb(infos)), models
260
+ return index(rows, models if labels else None, memory_gb(infos), sort), models
247
261
 
248
262
 
249
263
  def markdown(result, models):
tuieval/run_evals.py CHANGED
@@ -402,6 +402,8 @@ def cmd_pta(argv):
402
402
  p.add_argument("--only", help="comma-separated model labels (default: every model with results)")
403
403
  p.add_argument("--packs", help="comma-separated packs (default: all)")
404
404
  p.add_argument("--width", type=int, default=20, help="bar width in characters (default 20)")
405
+ p.add_argument("--sort", choices=("accuracy", "score"), default="accuracy",
406
+ help="order models by answers right (default) or by score (partial credit)")
405
407
  p.add_argument("--models")
406
408
  p.add_argument("--results-dir")
407
409
  a = p.parse_args(argv)
@@ -410,7 +412,7 @@ def cmd_pta(argv):
410
412
  e = engine.Engine(models_path=a.models, results_dir=a.results_dir)
411
413
  labels = [x.strip() for x in a.only.split(",")] if a.only else None
412
414
  packs = [x.strip() for x in a.packs.split(",")] if a.packs else None
413
- result, models = pta.for_engine(e, labels, packs)
415
+ result, models = pta.for_engine(e, labels, packs, sort=a.sort)
414
416
  out = Console(highlight=False)
415
417
  if not models:
416
418
  sys.exit("no finished results to compare" + (" for those models or packs" if labels or packs else ""))
tuieval/tui.py CHANGED
@@ -2892,6 +2892,10 @@ class ResultsScreen(Screen):
2892
2892
  def action_runs(self):
2893
2893
  self.app.open_sessions()
2894
2894
 
2895
+ FAIL_KINDS = [("Every kind", ""), ("Near misses", "near miss"), ("Partly right", "partly right"),
2896
+ ("Wrong", "wrong"), ("Cut off", "cut off"), ("No answer / format", "no answer / format"),
2897
+ ("Code crashed or hung", "code crashed or hung"), ("Critical", "critical")]
2898
+
2895
2899
  READINESS_HELP = ("[dim]PASS needs a Certify run with zero critical failures over enough trials and an "
2896
2900
  "accuracy lower bound that clears the pack's gate (pack.toml). Screening can only FAIL "
2897
2901
  "or look promising.[/dim]")
@@ -2913,6 +2917,8 @@ class ResultsScreen(Screen):
2913
2917
  with TabPane("PTA index", id="tab-pta"):
2914
2918
  with Horizontal(id="pta-controls"):
2915
2919
  yield Select([("All packs", "")], id="pta-pack", allow_blank=False, value="")
2920
+ yield Select([("By accuracy", "accuracy"), ("By score", "score")], id="pta-sort",
2921
+ allow_blank=False, value="accuracy")
2916
2922
  yield Button("Models: all", id="pta-models")
2917
2923
  yield Static("[dim]P parsimony (tokens): tokens used, reasoning and answer. T time: total time. "
2918
2924
  "A accuracy: answers right. Verdicts and critical failures are in Production "
@@ -2954,6 +2960,7 @@ class ResultsScreen(Screen):
2954
2960
  with TabPane("Failures", id="tab-failures"):
2955
2961
  with Horizontal(id="fail-controls"):
2956
2962
  yield Button("Models: all", id="fail-models")
2963
+ yield Select(self.FAIL_KINDS, id="fail-kind", allow_blank=False, value="")
2957
2964
  yield Static(id="fail-count")
2958
2965
  yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
2959
2966
  yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
@@ -3045,24 +3052,40 @@ class ResultsScreen(Screen):
3045
3052
  self.sort_tables()
3046
3053
 
3047
3054
  def fill_failures(self):
3048
- """Every wrong answer, of the models picked (Models: shared with the other tabs)."""
3055
+ """Every wrong answer, of the models picked (Models: shared with the other tabs), of the kind picked
3056
+ (insights.kind: near miss, partly right, wrong, cut off, …)."""
3057
+ from . import insights
3049
3058
  f = self.query_one("#failures", DataTable)
3050
3059
  f.clear(columns=True)
3051
- f.add_columns("Model", "Level", "Test", "Answered", "Reason")
3060
+ f.add_columns("Model", "Level", "Test", "Answered", "Kind", "Reason")
3052
3061
  every = {r["model"] for r in self.rows}
3053
3062
  chosen = {m for m in self.app.pq_models if m in every}
3054
- rows = [r for r in self.rows if not chosen or r["model"] in chosen]
3063
+ wanted = self.query_one("#fail-kind", Select).value or ""
3064
+ rows = [r for r in self.rows if (not chosen or r["model"] in chosen)
3065
+ and (r["ok"] or not wanted or insights.kind(r) == wanted)]
3055
3066
  # compare.failures lists the failed rows in order, so they line up with fail_rows
3056
3067
  self.fail_rows = [r for r in rows if not r["ok"]]
3068
+ colors = {k: c for k, c, _ in insights.KINDS}
3057
3069
  for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
3058
- when = dates.day(dates.answered_at(self.fail_rows[i].get("raw") or {}))
3059
- f.add_row(model, diff_badge(level), rich_escape(test), when, rich_escape(reason[:130]), key=str(i))
3070
+ r = self.fail_rows[i]
3071
+ when = dates.day(dates.answered_at(r.get("raw") or {}))
3072
+ k = insights.kind(r)
3073
+ checks = compare.checks_passed(r)
3074
+ label = f"{k} {checks[0]}/{checks[1]}" if checks and k in ("near miss", "partly right", "wrong") else k
3075
+ f.add_row(model, diff_badge(level), rich_escape(test), when, f"[{colors[k]}]{label}[/]",
3076
+ rich_escape(reason[:130]), key=str(i))
3060
3077
  self.query_one("#fail-models", Button).label = self.models_label(chosen)
3061
3078
  n = len(self.fail_rows)
3062
3079
  self.query_one("#fail-count", Static).update(
3063
- f"{n} failure{'s' * (n != 1)}" + (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
3080
+ f"{n} {'' if not wanted else wanted + ' '}failure{'s' * (n != 1)}"
3081
+ + (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
3064
3082
  self.sort_table(f)
3065
3083
 
3084
+ @on(Select.Changed, "#fail-kind")
3085
+ def fail_kind_changed(self):
3086
+ if getattr(self, "rows", None) is not None:
3087
+ self.fill_failures()
3088
+
3066
3089
  @on(DataTable.RowSelected, "#failures")
3067
3090
  def failure_selected(self, event):
3068
3091
  # in the order shown, so [ and ] step through them as sorted
@@ -3223,6 +3246,13 @@ class ResultsScreen(Screen):
3223
3246
  out = insights.panels(rows, models, e.packs, ref, bases, by, engine=e if main else None)
3224
3247
  body.update("\n\n".join(insights.render(p, i) for i, p in enumerate(out, 1)))
3225
3248
 
3249
+ @on(Select.Changed, "#pta-sort")
3250
+ def pta_sort_changed(self):
3251
+ """The Sort dropdown orders the bars and the table, taking over from a column sorted by a click."""
3252
+ self.app.result_sorts.pop("pta", None)
3253
+ if getattr(self, "rows", None) is not None:
3254
+ self.fill_pta()
3255
+
3226
3256
  @on(Select.Changed, "#pta-pack")
3227
3257
  @on(Select.Changed, "#pq-pack")
3228
3258
  def pack_changed(self, event):
@@ -3293,7 +3323,8 @@ class ResultsScreen(Screen):
3293
3323
  self.query_one("#pta-left-out", Static).update("")
3294
3324
  return
3295
3325
  # none picked: every model with enough answers (pta.index leaves out the rest, and says so)
3296
- result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos))
3326
+ result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos),
3327
+ self.query_one("#pta-sort", Select).value or "accuracy")
3297
3328
  self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
3298
3329
  self.pta_result = result
3299
3330
  removed = [m for m in self.pta_removed_now() if m in present]
@@ -3439,7 +3470,8 @@ class ResultsScreen(Screen):
3439
3470
  tt = r["total_secs"]
3440
3471
  cells.append([r["model"],
3441
3472
  hi(tt, ("-" if not tt else f"{tt:.1f}s" if tt < 60 else fmt_secs(tt)), "total_secs", "min"),
3442
- hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max"),
3473
+ hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max")
3474
+ + (f" [dim]{r['score']:.0%}[/dim]" if r["passed"] < r["n"] and r["score"] else ""),
3443
3475
  hi(r["med_tokens"], f"{r['med_tokens']:,.0f}" if r["med_tokens"] else "-", "med_tokens", "min"),
3444
3476
  hi(r["total_tokens"], f"{r['total_tokens']:,.0f}", "total_tokens", "min"),
3445
3477
  hi(r["tokens_per_pass"], f"{r['tokens_per_pass']:,.0f}" if r["tokens_per_pass"] else "-",
@@ -3469,7 +3501,7 @@ class ResultsScreen(Screen):
3469
3501
  "shared the machine's speed: compare their times with care.[/yellow]")
3470
3502
  title.update(line + " [dim]Time / Tokens: each question's median, added up. Med tok / Med s: median "
3471
3503
  "per answer. Tok/✓: tokens per answer right. Prompt t/s / Gen t/s: all tokens over all the time "
3472
- "they took (cached prompts left out). Fewest / Fastest: questions won; ties count for both. "
3504
+ "they took (cached prompts left out). Passed: questions right on every repeat, then the score (partial credit, e.g. checks passed) when some failed. Fewest / Fastest: questions won; ties count for both. "
3473
3505
  "Cut off: answers that hit max_tokens.[/dim]")
3474
3506
 
3475
3507
  @staticmethod
@@ -3478,6 +3510,9 @@ class ResultsScreen(Screen):
3478
3510
  else f"[yellow]{c['passed']}/{c['n']}[/yellow]")
3479
3511
  if c["n"] > 1 and c["passed"] in (0, c["n"]):
3480
3512
  mark += f"[dim]{c['n']}[/dim]"
3513
+ if c["passed"] < c["n"] and c.get("score"): # partial credit: how close it came
3514
+ checks = compare.checks_passed(c["rows"][0]) if c["n"] == 1 else None
3515
+ mark += f" [yellow]{checks[0]}/{checks[1]}[/yellow]" if checks else f" [yellow]{c['score']:.0%}[/yellow]"
3481
3516
  tok = c["tokens"]
3482
3517
  tok_s = "-" if not tok else (f"{tok / 1000:.1f}k" if tok >= 1000 else f"{tok:.0f}")
3483
3518
  if tok and best and tok == best and c["passed"]:
@@ -3624,6 +3659,7 @@ class EvalsApp(App):
3624
3659
  #ins-controls #ins-cost { width: 20; }
3625
3660
  #ins-body { height: 1fr; }
3626
3661
  #pta-controls Select { width: 28; margin-right: 1; }
3662
+ #pta-controls #pta-sort { width: 18; }
3627
3663
  #pta-bars { height: auto; padding: 1 0 0 0; }
3628
3664
  #pta { width: 1fr; height: auto; margin-top: 1; }
3629
3665
  #pta-left-out { height: auto; padding: 1 0 0 0; }
@@ -3632,6 +3668,7 @@ class EvalsApp(App):
3632
3668
  #pq-models { margin-right: 1; min-width: 20; }
3633
3669
  #fail-controls { height: 3; }
3634
3670
  #fail-models { margin-right: 1; min-width: 20; }
3671
+ #fail-kind { width: 28; margin-right: 1; }
3635
3672
  #fail-count { width: auto; padding: 1 1; }
3636
3673
  #pq-compare-title { padding: 0 1; height: auto; }
3637
3674
  #pq-summary { height: auto; } /* every picked model, no scrolling: the tab scrolls instead */
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: tuieval
3
- Version: 0.2.0.dev129
3
+ Version: 0.2.0.dev131
4
4
  Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
5
5
  Project-URL: Homepage, https://github.com/ashe-wb/tuieval
6
6
  Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -1,27 +1,27 @@
1
1
  tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
2
2
  tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
3
- tuieval/_version.py,sha256=AkC3QCfZjjCHBABegFa0MlKWs5cF5ey3LXR1zHC6wgU,537
3
+ tuieval/_version.py,sha256=biCEvo8zrqcKekhkznGrBJyvEyPtmcQdRlZacVAu4JU,537
4
4
  tuieval/agents.py,sha256=XmY4d4PAnC7BQcs_6KO9WTuIdpnwYVAdyhqItybWdAM,15408
5
5
  tuieval/builtin.toml,sha256=romPxP9o3r_4z3V8bdsmE9oUk0G2plDeqygPbqrwOys,4529
6
6
  tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
7
7
  tuieval/client.py,sha256=5wo0ySMD3o8up3cxKZicUWYFvOLZB60kYBV7yxP6LjI,12378
8
- tuieval/compare.py,sha256=WWnkGTm2F0-hhUG8EU-1TmhDdg3e3sFXo57kyU3dzck,36242
8
+ tuieval/compare.py,sha256=R_lGNSGPuDQ0j7-zuuniJtaRmI2KqDcDAhH82QO8Epc,36959
9
9
  tuieval/dates.py,sha256=CVazlwNjCRhD__D12zXLdeqF2ySt15X3xLUQeE-o6PY,6293
10
10
  tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
11
11
  tuieval/engine.py,sha256=M78wy43ZL4qmwUaZBDhpRZH5dWQKd-o0z59kWg7waCQ,136050
12
12
  tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
13
- tuieval/insights.py,sha256=ZIINv-iTFOomFK29pXF7xY_w-mf-_d4T1da4NuiJqiQ,28964
13
+ tuieval/insights.py,sha256=BDOn9UkDUkCKFcG2YIfZcnECgIFIUHxQawPRROGvDpM,29534
14
14
  tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
15
15
  tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
16
16
  tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
17
17
  tuieval/packs.py,sha256=KOj0BkwsB2124aslGyauZonyyGqtSrCJ-WAozEntmqE,17466
18
18
  tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
19
- tuieval/pta.py,sha256=BZbstnZFWpYmCdj9I0dbjXvLvhFhDCfB5vrRNWsAC2A,14493
19
+ tuieval/pta.py,sha256=Clfn9OBm85rHH1Ko7SAR7kIqhQdXhpEVN1nUhjVLTqQ,15480
20
20
  tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
21
- tuieval/run_evals.py,sha256=rbe1ae4gDUauoQMubfxLHfbokgN2OT683siuksdSeFo,50490
21
+ tuieval/run_evals.py,sha256=afZIzSdZPSCWnhc9qV_L1TXexl7YgSF6eKlV7RimMao,50679
22
22
  tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
23
23
  tuieval/selftest.py,sha256=cIGSDCrCK-QISAXjKSJDB4sD-WwS32bn3Gn4Y0oGoF8,11502
24
- tuieval/tui.py,sha256=9ckEs-Hvu5nqUi9FNQGB0ydiDJyoQVU8TeoMqxu05_Q,201891
24
+ tuieval/tui.py,sha256=XUxx7Z_b5ekfL0qdkQg4DD4Tra5VUObNvoORCA8mAhc,204245
25
25
  tuieval/tune.py,sha256=izz1Qs-d9PrWAqEZpyrLxfrlhlaouif9uFQSeJveL0k,41294
26
26
  tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
27
27
  tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -50,8 +50,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
50
50
  tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
51
51
  tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
52
52
  tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
53
- tuieval-0.2.0.dev129.dist-info/METADATA,sha256=66u5WjFPf7oURTEeNM8GVisRgnNh8ohFCEyg08G7AXc,15105
54
- tuieval-0.2.0.dev129.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
55
- tuieval-0.2.0.dev129.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
56
- tuieval-0.2.0.dev129.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
57
- tuieval-0.2.0.dev129.dist-info/RECORD,,
53
+ tuieval-0.2.0.dev131.dist-info/METADATA,sha256=Ok0YKjM61vA_ePXndEvbpY70zYpLJqQUqx5jGPdbV3c,15105
54
+ tuieval-0.2.0.dev131.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
55
+ tuieval-0.2.0.dev131.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
56
+ tuieval-0.2.0.dev131.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
57
+ tuieval-0.2.0.dev131.dist-info/RECORD,,