tuieval 0.2.0.dev129__py3-none-any.whl → 0.2.0.dev133__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tuieval/_version.py +2 -2
- tuieval/compare.py +23 -6
- tuieval/engine.py +8 -0
- tuieval/insights.py +12 -6
- tuieval/pta.py +50 -15
- tuieval/run_evals.py +3 -1
- tuieval/tui.py +51 -11
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev133.dist-info}/METADATA +1 -1
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev133.dist-info}/RECORD +12 -12
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev133.dist-info}/WHEEL +0 -0
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev133.dist-info}/entry_points.txt +0 -0
- {tuieval-0.2.0.dev129.dist-info → tuieval-0.2.0.dev133.dist-info}/licenses/LICENSE +0 -0
tuieval/_version.py
CHANGED
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.2.0.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 2, 0, '
|
|
21
|
+
__version__ = version = '0.2.0.dev133'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 2, 0, 'dev133')
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
tuieval/compare.py
CHANGED
|
@@ -248,12 +248,14 @@ def scorecard(rows, rng=None):
|
|
|
248
248
|
suites = sorted({r["suite"] for r in rows})
|
|
249
249
|
extra = dict(graders.COLUMNS) # columns your graders add (graders.scorecard_column)
|
|
250
250
|
over = any(r.get("over_budget") for r in rows) # a column only when some answer ran past max_tokens
|
|
251
|
-
header = ["model"] + suites + ["overall", "95% CI", "mean score", "all repeats", *extra, "errors", "trunc",
|
|
251
|
+
header = ["model", "repeats"] + suites + ["overall", "95% CI", "mean score", "all repeats", *extra, "errors", "trunc",
|
|
252
252
|
*(["over budget"] if over else []), "rep"]
|
|
253
253
|
table = []
|
|
254
254
|
for m in models:
|
|
255
255
|
mine = [r for r in rows if r["model"] == m]
|
|
256
|
-
|
|
256
|
+
per_test = collections.Counter(r["test"] for r in mine) # answers per question: the repeats
|
|
257
|
+
few, most = min(per_test.values()), max(per_test.values())
|
|
258
|
+
line = [m, f"×{few}" if few == most else f"×{few}-{most}"]
|
|
257
259
|
for s in suites:
|
|
258
260
|
sub = [r for r in mine if r["suite"] == s]
|
|
259
261
|
line.append(f"{sum(r['ok'] for r in sub)}/{len(sub)}" if sub else "-")
|
|
@@ -351,10 +353,22 @@ def speed(rows, infos, machine=None, speeds=None):
|
|
|
351
353
|
return header, table, frontier
|
|
352
354
|
|
|
353
355
|
|
|
356
|
+
NEAR_MISS = 0.8 # a failed answer scoring at least this (e.g. 5 of 6 checks) is a near miss
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def checks_passed(r):
|
|
360
|
+
"""(passed, total) checks of a failed code answer (from its stored checks, or its reason), else None."""
|
|
361
|
+
checks = (r.get("raw") or {}).get("checks")
|
|
362
|
+
if checks:
|
|
363
|
+
return sum(v is None for v in checks.values()), len(checks)
|
|
364
|
+
m = re.match(r"Failed (\d+)/(\d+) checks", r.get("reason") or "")
|
|
365
|
+
return (int(m[2]) - int(m[1]), int(m[2])) if m else None
|
|
366
|
+
|
|
367
|
+
|
|
354
368
|
def per_question(rows):
|
|
355
369
|
"""[{test, name, suite, difficulty, cells: {model: cell}, spread}] with one cell per model that
|
|
356
|
-
answered: n answers, passes, median tokens / seconds / tok/s over repeats,
|
|
357
|
-
partial (some answers from a pack still running), rows (the answers, for the detail view).
|
|
370
|
+
answered: n answers, passes, mean score (partial credit), median tokens / seconds / tok/s over repeats,
|
|
371
|
+
truncated count, partial (some answers from a pack still running), rows (the answers, for the detail view).
|
|
358
372
|
spread = {"tokens": .., "secs": ..}: largest / smallest model median (1 if one model)."""
|
|
359
373
|
tests = {}
|
|
360
374
|
for r in rows:
|
|
@@ -369,6 +383,7 @@ def per_question(rows):
|
|
|
369
383
|
c.update(rows=v, n=len(v), passed=sum(r["ok"] for r in v), truncated=sum(r["truncated"] for r in v),
|
|
370
384
|
over=sum(bool(r.get("over_budget")) for r in v),
|
|
371
385
|
partial=any(r.get("partial") for r in v),
|
|
386
|
+
score=statistics.fmean(r.get("score", r["ok"]) for r in v),
|
|
372
387
|
tokens=median(r["tokens"] or None for r in v), secs=median(r["latency"] / 1000 or None for r in v),
|
|
373
388
|
tps=median(r["gen_tps"] for r in v))
|
|
374
389
|
t["spread"] = _spread(t["cells"])
|
|
@@ -404,7 +419,7 @@ def complete_models(rows, suite, test_ids=None):
|
|
|
404
419
|
def compare_models(tests, models):
|
|
405
420
|
"""Per model over per_question entries that all `models` answered: passes, token and time
|
|
406
421
|
totals/medians (totals add each question's median over repeats, so repeats don't inflate them), questions where it used the fewest tokens / was fastest (among passing
|
|
407
|
-
answers for tokens; ties count for each), answers cut off."""
|
|
422
|
+
answers for tokens; ties count for each), answers cut off, mean score (partial credit)."""
|
|
408
423
|
out = []
|
|
409
424
|
for m in models:
|
|
410
425
|
cs = [t["cells"][m] for t in tests]
|
|
@@ -421,6 +436,7 @@ def compare_models(tests, models):
|
|
|
421
436
|
fastest += 1
|
|
422
437
|
rs = [r for c in cs for r in c["rows"]]
|
|
423
438
|
out.append({"model": m, "n": len(cs), "passed": passed, "total_tokens": total,
|
|
439
|
+
"score": statistics.fmean(c["score"] for c in cs) if cs else None,
|
|
424
440
|
"avg_pp_tps": _rate([(r.get("prompt_tokens"), r["prompt_tps"]) for r in rs]),
|
|
425
441
|
"avg_gen_tps": _rate([(r["tokens"], r["gen_tps"]) for r in rs]),
|
|
426
442
|
"med_tokens": median(c["tokens"] for c in cs), "med_secs": median(c["secs"] for c in cs),
|
|
@@ -664,7 +680,8 @@ def main(argv):
|
|
|
664
680
|
for note in settings_notes(infos):
|
|
665
681
|
print("NOTE:", note)
|
|
666
682
|
print_table(*scorecard(rows, rng))
|
|
667
|
-
print("\
|
|
683
|
+
print("\nrepeats = answers per question (×1: one answer each, the roughest accuracy).")
|
|
684
|
+
print("mean score = average score, with partial credit; all repeats = tests passed on every repeat.")
|
|
668
685
|
print("trunc = hit max_tokens; rep = output ends in a degenerate repeated pattern.")
|
|
669
686
|
header, table = by_difficulty(rows)
|
|
670
687
|
if any(r.get("difficulty") in LEVELS for r in rows):
|
tuieval/engine.py
CHANGED
|
@@ -989,6 +989,14 @@ class Engine:
|
|
|
989
989
|
need = {(t["id"], r) for t in pack.tests for r in range(self.certify_repeat(pack))}
|
|
990
990
|
return "certified" if need <= keys else "screened"
|
|
991
991
|
|
|
992
|
+
def result_repeats(self, label, pack_name, tier="certify"):
|
|
993
|
+
"""How many times each answered question of a stored result was asked: (fewest, most), or None."""
|
|
994
|
+
data = self._result(self.result_path(label, pack_name, tier))
|
|
995
|
+
if not data or not data["results"]:
|
|
996
|
+
return None
|
|
997
|
+
per_test = collections.Counter(r["test"] for r in data["results"])
|
|
998
|
+
return min(per_test.values()), max(per_test.values())
|
|
999
|
+
|
|
992
1000
|
def _partial_lines(self, path):
|
|
993
1001
|
"""(header, answers) of an unfinished run's .partial.jsonl, cached like _result;
|
|
994
1002
|
(None, []) if there is none."""
|
tuieval/insights.py
CHANGED
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
2 use cases accuracy per model and use case (pack group) as a heatmap; the best marked
|
|
7
7
|
3 vs base a fine-tune against its base model (models.toml `base`): per pack, and the question
|
|
8
8
|
categories it gained or lost most
|
|
9
|
-
4 why they fail each model's answers split into right,
|
|
10
|
-
crashed or hung, and
|
|
9
|
+
4 why they fail each model's answers split into right, near miss (most checks passed), partly right,
|
|
10
|
+
wrong, cut off (or out of time), no answer / bad format, code crashed or hung, and
|
|
11
|
+
critical failures
|
|
11
12
|
5 accuracy / cost accuracy against median tokens (or seconds) per question; the models no other beats
|
|
12
13
|
on both
|
|
13
14
|
6 questions questions every model fails (check them), every model passes (no signal), and the
|
|
@@ -267,7 +268,8 @@ def categories(rows, m, base, min_q=2):
|
|
|
267
268
|
|
|
268
269
|
|
|
269
270
|
# ---------------------------------------------------------------- 4 why they fail
|
|
270
|
-
KINDS = (("right", "green", "█"), ("
|
|
271
|
+
KINDS = (("right", "green", "█"), ("near miss", "#87af5f", "█"), ("partly right", "#af875f", "█"),
|
|
272
|
+
("wrong", "red", "█"), ("cut off", "yellow", "█"),
|
|
271
273
|
("no answer / format", "magenta", "█"), ("code crashed or hung", "cyan", "█"), ("critical", "#ff5faf", "▓"))
|
|
272
274
|
FORMAT = ("No 'ANSWER:' line", "Empty answer", "No code", "no tool call")
|
|
273
275
|
CRASHED = ("Timed out after", "Harness crashed", "grader error")
|
|
@@ -278,13 +280,15 @@ def kind(r):
|
|
|
278
280
|
return "right"
|
|
279
281
|
if r.get("severity") == "critical":
|
|
280
282
|
return "critical"
|
|
281
|
-
if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET")):
|
|
283
|
+
if r["truncated"] or r["reason"].startswith(("TRUNCATED", "OVER BUDGET", "time limit")):
|
|
282
284
|
return "cut off"
|
|
283
285
|
if r["reason"].startswith(FORMAT):
|
|
284
286
|
return "no answer / format"
|
|
285
287
|
if r["reason"].startswith(CRASHED):
|
|
286
288
|
return "code crashed or hung"
|
|
287
|
-
|
|
289
|
+
# partial credit (e.g. checks passed): close isn't the same as all wrong
|
|
290
|
+
score = r.get("score") or 0
|
|
291
|
+
return "near miss" if score >= compare.NEAR_MISS else "partly right" if score > 0 else "wrong"
|
|
288
292
|
|
|
289
293
|
|
|
290
294
|
def _split(counts, total, width):
|
|
@@ -322,7 +326,9 @@ def failure_mix(rows, models, width=36):
|
|
|
322
326
|
top, n = max(((k, c[k]) for k, *_ in KINDS[1:-1]), key=lambda x: x[1])
|
|
323
327
|
if n / fails >= 0.4 and top != "wrong":
|
|
324
328
|
actions.append(f"{m}: {n} of its {fails} failures are {top}: " + {
|
|
325
|
-
"cut off": "it thinks too long for max_tokens, rather than reasoning wrong.",
|
|
329
|
+
"cut off": "it thinks too long for max_tokens (or the time limit), rather than reasoning wrong.",
|
|
330
|
+
"near miss": "they pass most of their checks: it's closer than its accuracy shows.",
|
|
331
|
+
"partly right": "they pass some of their checks, not none.",
|
|
326
332
|
"no answer / format": "it doesn't follow the answer format (check its chat template and the "
|
|
327
333
|
"pack's instructions).",
|
|
328
334
|
"code crashed or hung": "its code crashes or never finishes."}[top])
|
tuieval/pta.py
CHANGED
|
@@ -7,11 +7,17 @@
|
|
|
7
7
|
model's ("28×").
|
|
8
8
|
A accuracy the share of answers to those questions that passed (%).
|
|
9
9
|
|
|
10
|
+
Beside A, the score: partial credit averaged over the same answers (e.g. a code answer passing 5 of 6
|
|
11
|
+
checks scores 83%, where A counts it wrong). Shown when it differs from A; models can be ordered by it.
|
|
12
|
+
|
|
10
13
|
The P and T bars are as long as the tokens and time used (the model that used the most fills its bar,
|
|
11
14
|
the others in proportion, at least one block), so shorter is better; the A bar: longer is better.
|
|
12
15
|
P and T also carry a 0-100 score (100 for the best, POINTS_PER_DOUBLING points less for every
|
|
13
16
|
doubling), used to order models with the same accuracy.
|
|
14
17
|
|
|
18
|
+
Beside each name, the repeats: how many times it answered each of those questions ("×3", or "×1-3" when
|
|
19
|
+
it varies). A ×1 model's accuracy rests on one answer per question, so it is rougher than a ×3 one's.
|
|
20
|
+
|
|
15
21
|
Models are compared on the questions all of them answered, so one that skipped some never looks
|
|
16
22
|
faster. Server and connection errors aren't answers. By default a model with fewer than half as many
|
|
17
23
|
answers as the most-answered model is left out (listed, so it can be picked), or it would shrink the
|
|
@@ -46,11 +52,15 @@ def memory_gb(infos):
|
|
|
46
52
|
return out
|
|
47
53
|
|
|
48
54
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
55
|
+
SORTS = ("accuracy", "score")
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def index(rows, models=None, memory=None, sort="accuracy"):
|
|
59
|
+
"""{"questions": n shared, "models": [{model, P, T, A, S, cut, cut_pct, tokens, total_s, passed, answers, alongside,
|
|
60
|
+
fewest, fastest, tps, ttft, memory_gb, repeats: (fewest, most) answers per question}],
|
|
52
61
|
"left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
|
|
53
|
-
most-answered model}, best accuracy
|
|
62
|
+
most-answered model}, best accuracy (sort="accuracy") or score (sort="score") first. S: the mean
|
|
63
|
+
score (partial credit) in %. models: the ones to compare (default: every model in rows
|
|
54
64
|
with enough answers; models named here are compared whatever their count). memory: memory_gb()."""
|
|
55
65
|
no_answers = sorted({r["model"] for r in rows} - {r["model"] for r in rows if not r.get("error")})
|
|
56
66
|
rows = [r for r in rows if not r.get("error")] # a server or connection error isn't an answer
|
|
@@ -75,6 +85,7 @@ def index(rows, models=None, memory=None):
|
|
|
75
85
|
"total_s": sum(c["secs"] or 0 for c in cells),
|
|
76
86
|
"passed": sum(c["passed"] for c in cells), "answers": answers,
|
|
77
87
|
"A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
|
|
88
|
+
"S": 100 * sum(c["score"] * c["n"] for c in cells) / answers if answers else None,
|
|
78
89
|
# failed only because max_tokens ran out (overthinking, not wrong yet)
|
|
79
90
|
"cut": sum(c["truncated"] for c in cells),
|
|
80
91
|
"cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
|
|
@@ -84,6 +95,7 @@ def index(rows, models=None, memory=None):
|
|
|
84
95
|
"tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
|
|
85
96
|
"ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
|
|
86
97
|
"memory_gb": (memory or {}).get(m),
|
|
98
|
+
"repeats": (min(c["n"] for c in cells), max(c["n"] for c in cells)) if cells else None,
|
|
87
99
|
# answers that ran while other models were served shared the machine's speed
|
|
88
100
|
"alongside": any(r.get("alongside") for c in cells for r in c["rows"])})
|
|
89
101
|
fastest = min((x["total_s"] for x in out if x["total_s"] > 0), default=None)
|
|
@@ -95,7 +107,8 @@ def index(rows, models=None, memory=None):
|
|
|
95
107
|
x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
|
|
96
108
|
x["fewest"] = bool(fewest) and x["tokens"] == fewest
|
|
97
109
|
x["fastest"] = bool(fastest) and x["total_s"] == fastest
|
|
98
|
-
|
|
110
|
+
first, then = ("S", "A") if sort == "score" else ("A", "S")
|
|
111
|
+
out.sort(key=lambda x: (-(x[first] or 0), -(x[then] or 0), -(x["T"] or 0), x["model"]))
|
|
99
112
|
return {"questions": len(shared), "models": out, "left_out": left_out, "no_answers": no_answers,
|
|
100
113
|
"most": most}
|
|
101
114
|
|
|
@@ -107,6 +120,14 @@ LETTERS = (("P", "parsimony (tokens)", "▼ shorter is better", "tokens", "token
|
|
|
107
120
|
("A", "accuracy", "▲ longer is better", "A", "A", "green"))
|
|
108
121
|
|
|
109
122
|
|
|
123
|
+
def repeats(x):
|
|
124
|
+
"""How many times a model answered each question compared: "×3", "×1-3", or "" when unknown."""
|
|
125
|
+
r = x.get("repeats")
|
|
126
|
+
if not r:
|
|
127
|
+
return ""
|
|
128
|
+
return f"×{r[0]}" if r[0] == r[1] else f"×{r[0]}-{r[1]}"
|
|
129
|
+
|
|
130
|
+
|
|
110
131
|
def times(v):
|
|
111
132
|
"""A multiple of the best: "1.0×", "2.4×", "28×"."""
|
|
112
133
|
return "-" if v is None else f"{v:.1f}×" if v < 9.95 else f"{v:.0f}×"
|
|
@@ -129,6 +150,8 @@ def bars(result, width=20, markup=True, click=None):
|
|
|
129
150
|
if not result["models"]:
|
|
130
151
|
return []
|
|
131
152
|
name_w = max(len(x["model"]) for x in result["models"])
|
|
153
|
+
rep_w = max(len(repeats(x)) for x in result["models"])
|
|
154
|
+
name_w += rep_w + 1 if rep_w else 0 # the repeats sit after the name, in its column
|
|
132
155
|
# each column: the bar, a space, "100%" or "9.8×" (6), then a gap; at least as wide as its heading
|
|
133
156
|
cell_w = max(width + 9, *(len(t) + 2 for k, name, better, *_ in LETTERS for t in (f"{k} {name}", better)))
|
|
134
157
|
pad = " " * (name_w + 2)
|
|
@@ -145,6 +168,10 @@ def bars(result, width=20, markup=True, click=None):
|
|
|
145
168
|
if v is not None and v < 100: # full only when every answer passed
|
|
146
169
|
full = min(full, width - 1)
|
|
147
170
|
num = "-" if v is None else compare.pct(v / 100)
|
|
171
|
+
score = x.get("S")
|
|
172
|
+
if score is not None and v is not None and compare.pct(score / 100) != num:
|
|
173
|
+
note = f"score {compare.pct(score / 100)}"
|
|
174
|
+
num = f"{num:>6} " + (f"[dim]{note}[/dim]" if markup else note)
|
|
148
175
|
cut = x.get("cut_pct") or 0
|
|
149
176
|
part = min(width - full, max(1, round(width * cut / 100))) if cut else 0
|
|
150
177
|
if cut: # after the accuracy, which stays aligned with the other rows'
|
|
@@ -160,11 +187,14 @@ def bars(result, width=20, markup=True, click=None):
|
|
|
160
187
|
bar = (f"[{color}]{bar}[/]" + (f"[#ff8c00]{'█' * part}[/]" if part else "") + f"[grey37]{rest}[/]"
|
|
161
188
|
if markup else bar + cut_bar + rest)
|
|
162
189
|
cells.append(f"{bar} {num:>6}" + " " * max(0, cell_w - width - 1 - max(6, len(num))))
|
|
163
|
-
|
|
190
|
+
tag = repeats(x)
|
|
191
|
+
name = f"{x['model'] + (' ' + tag if tag else ''):<{name_w}}"
|
|
164
192
|
if markup:
|
|
165
|
-
pad_w = name_w - len(x["model"])
|
|
166
|
-
|
|
167
|
-
|
|
193
|
+
pad_w = name_w - len(x["model"]) - (len(tag) + 1 if tag else 0)
|
|
194
|
+
# a single answer per question in yellow: its accuracy is the roughest
|
|
195
|
+
tag = f" [{'yellow' if x['repeats'][1] == 1 else 'dim'}]{tag}[/]" if tag else ""
|
|
196
|
+
name = ((f"[@click={click}({row})]{rich_escape(x['model'])}[/]" if click else rich_escape(x["model"]))
|
|
197
|
+
+ tag + " " * pad_w)
|
|
168
198
|
lines.append((f"{name} " + "".join(cells)).rstrip())
|
|
169
199
|
return lines
|
|
170
200
|
|
|
@@ -189,11 +219,12 @@ def table(result):
|
|
|
189
219
|
"""(header, rows) for a plain table of the index."""
|
|
190
220
|
n = result["questions"] or 1
|
|
191
221
|
over = any(x.get("over") for x in result["models"]) # a column only when some answer ran past max_tokens
|
|
192
|
-
header = ["model", "tokens/answer", "time/answer", "answers right", "truncated", *(["over budget"] if over else []),
|
|
222
|
+
header = ["model", "repeats", "tokens/answer", "time/answer", "answers right", "score", "truncated", *(["over budget"] if over else []),
|
|
193
223
|
"tok/s", "TTFT s", "memory GB"]
|
|
194
|
-
rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
|
|
224
|
+
rows = [[x["model"], repeats(x) or "-", f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
|
|
195
225
|
fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
|
|
196
|
-
f"{x['passed']}/{x['answers']}",
|
|
226
|
+
f"{x['passed']}/{x['answers']}", "-" if x.get("S") is None else compare.pct(x["S"] / 100),
|
|
227
|
+
f"{x.get('cut') or 0}/{x['answers']}",
|
|
197
228
|
*([f"{x.get('over') or 0}/{x['answers']}"] if over else []),
|
|
198
229
|
_num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
|
|
199
230
|
_num(x.get("memory_gb"), ".1f")] for x in result["models"]]
|
|
@@ -228,7 +259,11 @@ def scope_note(result, models):
|
|
|
228
259
|
return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
|
|
229
260
|
"P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
|
|
230
261
|
"and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
|
|
231
|
-
"(longer is better)"
|
|
262
|
+
"(longer is better); score: partial credit, e.g. 5 of 6 checks passed is 83% (shown when it differs)"
|
|
263
|
+
+ (f"; ×n beside a name: answers per question. ×1 models answered each question once, so one "
|
|
264
|
+
f"answer more or less right moves their accuracy by {100 / result['questions']:.0f} points; "
|
|
265
|
+
"repeats steady it" if len({x.get("repeats") for x in result["models"]}) > 1
|
|
266
|
+
or any(x.get("repeats") and x["repeats"][1] == 1 for x in result["models"]) else "")
|
|
232
267
|
+ ("; orange, (n% truncated): answers cut off at max_tokens. They count as wrong, but the model ran "
|
|
233
268
|
"out of tokens rather than answering wrong." if any(x.get("cut") for x in result["models"]) else ".")
|
|
234
269
|
+ (" Over budget: answers that used more tokens than the max_tokens sent, because the provider didn't "
|
|
@@ -236,14 +271,14 @@ def scope_note(result, models):
|
|
|
236
271
|
if any(x.get("over") for x in result["models"]) else ""))
|
|
237
272
|
|
|
238
273
|
|
|
239
|
-
def for_engine(e, labels=None, packs=None, results_dir=None):
|
|
274
|
+
def for_engine(e, labels=None, packs=None, results_dir=None, sort="accuracy"):
|
|
240
275
|
"""(result, models) from a workspace's finished results (results on their way out excluded)."""
|
|
241
276
|
paths = [p for p in compare.default_paths(results_dir or e.results_dir) if not e.superseded(p)]
|
|
242
277
|
rows, infos = compare.load(paths) if paths else ([], [])
|
|
243
278
|
if packs:
|
|
244
279
|
rows = [r for r in rows if r["suite"] in packs]
|
|
245
280
|
models = sorted({r["model"] for r in rows} & set(labels) if labels else {r["model"] for r in rows})
|
|
246
|
-
return index(rows, models if labels else None, memory_gb(infos)), models
|
|
281
|
+
return index(rows, models if labels else None, memory_gb(infos), sort), models
|
|
247
282
|
|
|
248
283
|
|
|
249
284
|
def markdown(result, models):
|
tuieval/run_evals.py
CHANGED
|
@@ -402,6 +402,8 @@ def cmd_pta(argv):
|
|
|
402
402
|
p.add_argument("--only", help="comma-separated model labels (default: every model with results)")
|
|
403
403
|
p.add_argument("--packs", help="comma-separated packs (default: all)")
|
|
404
404
|
p.add_argument("--width", type=int, default=20, help="bar width in characters (default 20)")
|
|
405
|
+
p.add_argument("--sort", choices=("accuracy", "score"), default="accuracy",
|
|
406
|
+
help="order models by answers right (default) or by score (partial credit)")
|
|
405
407
|
p.add_argument("--models")
|
|
406
408
|
p.add_argument("--results-dir")
|
|
407
409
|
a = p.parse_args(argv)
|
|
@@ -410,7 +412,7 @@ def cmd_pta(argv):
|
|
|
410
412
|
e = engine.Engine(models_path=a.models, results_dir=a.results_dir)
|
|
411
413
|
labels = [x.strip() for x in a.only.split(",")] if a.only else None
|
|
412
414
|
packs = [x.strip() for x in a.packs.split(",")] if a.packs else None
|
|
413
|
-
result, models = pta.for_engine(e, labels, packs)
|
|
415
|
+
result, models = pta.for_engine(e, labels, packs, sort=a.sort)
|
|
414
416
|
out = Console(highlight=False)
|
|
415
417
|
if not models:
|
|
416
418
|
sys.exit("no finished results to compare" + (" for those models or packs" if labels or packs else ""))
|
tuieval/tui.py
CHANGED
|
@@ -937,7 +937,10 @@ class SetupScreen(Screen):
|
|
|
937
937
|
for n in e.packs:
|
|
938
938
|
st = e.result_status(label, n)
|
|
939
939
|
if st:
|
|
940
|
-
|
|
940
|
+
reps = e.result_repeats(label, n) if st in ("certified", "screened") else None
|
|
941
|
+
times = ("" if not reps else f"[dim]×{reps[0]}[/dim]" if reps[0] == reps[1]
|
|
942
|
+
else f"[dim]×{reps[0]}-{reps[1]}[/dim]")
|
|
943
|
+
packs.append(f"{mark.get(st, '[yellow]~[/yellow]')}{n}{times}")
|
|
941
944
|
if st.startswith("outdated"):
|
|
942
945
|
why.setdefault(st.removeprefix("outdated: "), []).append(n)
|
|
943
946
|
if why: # each reason once, not once per pack
|
|
@@ -951,7 +954,7 @@ class SetupScreen(Screen):
|
|
|
951
954
|
hidden = (f" [cyan]{' '.join(tags)}[/cyan]" if tags else "") + hidden
|
|
952
955
|
detail.update(f"[b]{label}[/b] [dim]{m['server']} · {m.get('model') or ''}[/dim]"
|
|
953
956
|
f"{serving_badge(e, m, not e.has_results())}{hidden}\n{verdicts}\n"
|
|
954
|
-
+ ("[dim]results (✓ certified ◐ screened ~ outdated … partial):[/dim] " + " ".join(packs)
|
|
957
|
+
+ ("[dim]results (✓ certified ◐ screened ~ outdated … partial; ×n answers per question):[/dim] " + " ".join(packs)
|
|
955
958
|
if packs else "[dim]no results yet[/dim]"))
|
|
956
959
|
|
|
957
960
|
@on(Checkbox.Changed, "#show-hidden")
|
|
@@ -2892,6 +2895,10 @@ class ResultsScreen(Screen):
|
|
|
2892
2895
|
def action_runs(self):
|
|
2893
2896
|
self.app.open_sessions()
|
|
2894
2897
|
|
|
2898
|
+
FAIL_KINDS = [("Every kind", ""), ("Near misses", "near miss"), ("Partly right", "partly right"),
|
|
2899
|
+
("Wrong", "wrong"), ("Cut off", "cut off"), ("No answer / format", "no answer / format"),
|
|
2900
|
+
("Code crashed or hung", "code crashed or hung"), ("Critical", "critical")]
|
|
2901
|
+
|
|
2895
2902
|
READINESS_HELP = ("[dim]PASS needs a Certify run with zero critical failures over enough trials and an "
|
|
2896
2903
|
"accuracy lower bound that clears the pack's gate (pack.toml). Screening can only FAIL "
|
|
2897
2904
|
"or look promising.[/dim]")
|
|
@@ -2913,6 +2920,8 @@ class ResultsScreen(Screen):
|
|
|
2913
2920
|
with TabPane("PTA index", id="tab-pta"):
|
|
2914
2921
|
with Horizontal(id="pta-controls"):
|
|
2915
2922
|
yield Select([("All packs", "")], id="pta-pack", allow_blank=False, value="")
|
|
2923
|
+
yield Select([("By accuracy", "accuracy"), ("By score", "score")], id="pta-sort",
|
|
2924
|
+
allow_blank=False, value="accuracy")
|
|
2916
2925
|
yield Button("Models: all", id="pta-models")
|
|
2917
2926
|
yield Static("[dim]P parsimony (tokens): tokens used, reasoning and answer. T time: total time. "
|
|
2918
2927
|
"A accuracy: answers right. Verdicts and critical failures are in Production "
|
|
@@ -2954,6 +2963,7 @@ class ResultsScreen(Screen):
|
|
|
2954
2963
|
with TabPane("Failures", id="tab-failures"):
|
|
2955
2964
|
with Horizontal(id="fail-controls"):
|
|
2956
2965
|
yield Button("Models: all", id="fail-models")
|
|
2966
|
+
yield Select(self.FAIL_KINDS, id="fail-kind", allow_blank=False, value="")
|
|
2957
2967
|
yield Static(id="fail-count")
|
|
2958
2968
|
yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
|
|
2959
2969
|
yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
|
|
@@ -3045,24 +3055,40 @@ class ResultsScreen(Screen):
|
|
|
3045
3055
|
self.sort_tables()
|
|
3046
3056
|
|
|
3047
3057
|
def fill_failures(self):
|
|
3048
|
-
"""Every wrong answer, of the models picked (Models: shared with the other tabs)
|
|
3058
|
+
"""Every wrong answer, of the models picked (Models: shared with the other tabs), of the kind picked
|
|
3059
|
+
(insights.kind: near miss, partly right, wrong, cut off, …)."""
|
|
3060
|
+
from . import insights
|
|
3049
3061
|
f = self.query_one("#failures", DataTable)
|
|
3050
3062
|
f.clear(columns=True)
|
|
3051
|
-
f.add_columns("Model", "Level", "Test", "Answered", "Reason")
|
|
3063
|
+
f.add_columns("Model", "Level", "Test", "Answered", "Kind", "Reason")
|
|
3052
3064
|
every = {r["model"] for r in self.rows}
|
|
3053
3065
|
chosen = {m for m in self.app.pq_models if m in every}
|
|
3054
|
-
|
|
3066
|
+
wanted = self.query_one("#fail-kind", Select).value or ""
|
|
3067
|
+
rows = [r for r in self.rows if (not chosen or r["model"] in chosen)
|
|
3068
|
+
and (r["ok"] or not wanted or insights.kind(r) == wanted)]
|
|
3055
3069
|
# compare.failures lists the failed rows in order, so they line up with fail_rows
|
|
3056
3070
|
self.fail_rows = [r for r in rows if not r["ok"]]
|
|
3071
|
+
colors = {k: c for k, c, _ in insights.KINDS}
|
|
3057
3072
|
for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
|
|
3058
|
-
|
|
3059
|
-
|
|
3073
|
+
r = self.fail_rows[i]
|
|
3074
|
+
when = dates.day(dates.answered_at(r.get("raw") or {}))
|
|
3075
|
+
k = insights.kind(r)
|
|
3076
|
+
checks = compare.checks_passed(r)
|
|
3077
|
+
label = f"{k} {checks[0]}/{checks[1]}" if checks and k in ("near miss", "partly right", "wrong") else k
|
|
3078
|
+
f.add_row(model, diff_badge(level), rich_escape(test), when, f"[{colors[k]}]{label}[/]",
|
|
3079
|
+
rich_escape(reason[:130]), key=str(i))
|
|
3060
3080
|
self.query_one("#fail-models", Button).label = self.models_label(chosen)
|
|
3061
3081
|
n = len(self.fail_rows)
|
|
3062
3082
|
self.query_one("#fail-count", Static).update(
|
|
3063
|
-
f"{n}
|
|
3083
|
+
f"{n} {'' if not wanted else wanted + ' '}failure{'s' * (n != 1)}"
|
|
3084
|
+
+ (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
|
|
3064
3085
|
self.sort_table(f)
|
|
3065
3086
|
|
|
3087
|
+
@on(Select.Changed, "#fail-kind")
|
|
3088
|
+
def fail_kind_changed(self):
|
|
3089
|
+
if getattr(self, "rows", None) is not None:
|
|
3090
|
+
self.fill_failures()
|
|
3091
|
+
|
|
3066
3092
|
@on(DataTable.RowSelected, "#failures")
|
|
3067
3093
|
def failure_selected(self, event):
|
|
3068
3094
|
# in the order shown, so [ and ] step through them as sorted
|
|
@@ -3223,6 +3249,13 @@ class ResultsScreen(Screen):
|
|
|
3223
3249
|
out = insights.panels(rows, models, e.packs, ref, bases, by, engine=e if main else None)
|
|
3224
3250
|
body.update("\n\n".join(insights.render(p, i) for i, p in enumerate(out, 1)))
|
|
3225
3251
|
|
|
3252
|
+
@on(Select.Changed, "#pta-sort")
|
|
3253
|
+
def pta_sort_changed(self):
|
|
3254
|
+
"""The Sort dropdown orders the bars and the table, taking over from a column sorted by a click."""
|
|
3255
|
+
self.app.result_sorts.pop("pta", None)
|
|
3256
|
+
if getattr(self, "rows", None) is not None:
|
|
3257
|
+
self.fill_pta()
|
|
3258
|
+
|
|
3226
3259
|
@on(Select.Changed, "#pta-pack")
|
|
3227
3260
|
@on(Select.Changed, "#pq-pack")
|
|
3228
3261
|
def pack_changed(self, event):
|
|
@@ -3293,7 +3326,8 @@ class ResultsScreen(Screen):
|
|
|
3293
3326
|
self.query_one("#pta-left-out", Static).update("")
|
|
3294
3327
|
return
|
|
3295
3328
|
# none picked: every model with enough answers (pta.index leaves out the rest, and says so)
|
|
3296
|
-
result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos)
|
|
3329
|
+
result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos),
|
|
3330
|
+
self.query_one("#pta-sort", Select).value or "accuracy")
|
|
3297
3331
|
self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
|
|
3298
3332
|
self.pta_result = result
|
|
3299
3333
|
removed = [m for m in self.pta_removed_now() if m in present]
|
|
@@ -3439,7 +3473,8 @@ class ResultsScreen(Screen):
|
|
|
3439
3473
|
tt = r["total_secs"]
|
|
3440
3474
|
cells.append([r["model"],
|
|
3441
3475
|
hi(tt, ("-" if not tt else f"{tt:.1f}s" if tt < 60 else fmt_secs(tt)), "total_secs", "min"),
|
|
3442
|
-
hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max")
|
|
3476
|
+
hi(r["passed"], f"{r['passed']}/{r['n']}", "passed", "max")
|
|
3477
|
+
+ (f" [dim]{r['score']:.0%}[/dim]" if r["passed"] < r["n"] and r["score"] else ""),
|
|
3443
3478
|
hi(r["med_tokens"], f"{r['med_tokens']:,.0f}" if r["med_tokens"] else "-", "med_tokens", "min"),
|
|
3444
3479
|
hi(r["total_tokens"], f"{r['total_tokens']:,.0f}", "total_tokens", "min"),
|
|
3445
3480
|
hi(r["tokens_per_pass"], f"{r['tokens_per_pass']:,.0f}" if r["tokens_per_pass"] else "-",
|
|
@@ -3469,7 +3504,7 @@ class ResultsScreen(Screen):
|
|
|
3469
3504
|
"shared the machine's speed: compare their times with care.[/yellow]")
|
|
3470
3505
|
title.update(line + " [dim]Time / Tokens: each question's median, added up. Med tok / Med s: median "
|
|
3471
3506
|
"per answer. Tok/✓: tokens per answer right. Prompt t/s / Gen t/s: all tokens over all the time "
|
|
3472
|
-
"they took (cached prompts left out). Fewest / Fastest: questions won; ties count for both. "
|
|
3507
|
+
"they took (cached prompts left out). Passed: questions right on every repeat, then the score (partial credit, e.g. checks passed) when some failed. Fewest / Fastest: questions won; ties count for both. "
|
|
3473
3508
|
"Cut off: answers that hit max_tokens.[/dim]")
|
|
3474
3509
|
|
|
3475
3510
|
@staticmethod
|
|
@@ -3478,6 +3513,9 @@ class ResultsScreen(Screen):
|
|
|
3478
3513
|
else f"[yellow]{c['passed']}/{c['n']}[/yellow]")
|
|
3479
3514
|
if c["n"] > 1 and c["passed"] in (0, c["n"]):
|
|
3480
3515
|
mark += f"[dim]{c['n']}[/dim]"
|
|
3516
|
+
if c["passed"] < c["n"] and c.get("score"): # partial credit: how close it came
|
|
3517
|
+
checks = compare.checks_passed(c["rows"][0]) if c["n"] == 1 else None
|
|
3518
|
+
mark += f" [yellow]{checks[0]}/{checks[1]}[/yellow]" if checks else f" [yellow]{c['score']:.0%}[/yellow]"
|
|
3481
3519
|
tok = c["tokens"]
|
|
3482
3520
|
tok_s = "-" if not tok else (f"{tok / 1000:.1f}k" if tok >= 1000 else f"{tok:.0f}")
|
|
3483
3521
|
if tok and best and tok == best and c["passed"]:
|
|
@@ -3624,6 +3662,7 @@ class EvalsApp(App):
|
|
|
3624
3662
|
#ins-controls #ins-cost { width: 20; }
|
|
3625
3663
|
#ins-body { height: 1fr; }
|
|
3626
3664
|
#pta-controls Select { width: 28; margin-right: 1; }
|
|
3665
|
+
#pta-controls #pta-sort { width: 18; }
|
|
3627
3666
|
#pta-bars { height: auto; padding: 1 0 0 0; }
|
|
3628
3667
|
#pta { width: 1fr; height: auto; margin-top: 1; }
|
|
3629
3668
|
#pta-left-out { height: auto; padding: 1 0 0 0; }
|
|
@@ -3632,6 +3671,7 @@ class EvalsApp(App):
|
|
|
3632
3671
|
#pq-models { margin-right: 1; min-width: 20; }
|
|
3633
3672
|
#fail-controls { height: 3; }
|
|
3634
3673
|
#fail-models { margin-right: 1; min-width: 20; }
|
|
3674
|
+
#fail-kind { width: 28; margin-right: 1; }
|
|
3635
3675
|
#fail-count { width: auto; padding: 1 1; }
|
|
3636
3676
|
#pq-compare-title { padding: 0 1; height: auto; }
|
|
3637
3677
|
#pq-summary { height: auto; } /* every picked model, no scrolling: the tab scrolls instead */
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: tuieval
|
|
3
|
-
Version: 0.2.0.
|
|
3
|
+
Version: 0.2.0.dev133
|
|
4
4
|
Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
|
|
5
5
|
Project-URL: Homepage, https://github.com/ashe-wb/tuieval
|
|
6
6
|
Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
|
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
|
|
2
2
|
tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
|
|
3
|
-
tuieval/_version.py,sha256=
|
|
3
|
+
tuieval/_version.py,sha256=seXDvsrsCa7COI9ADByO7-ByiyH-SLyoBwimkvhuD0c,537
|
|
4
4
|
tuieval/agents.py,sha256=XmY4d4PAnC7BQcs_6KO9WTuIdpnwYVAdyhqItybWdAM,15408
|
|
5
5
|
tuieval/builtin.toml,sha256=romPxP9o3r_4z3V8bdsmE9oUk0G2plDeqygPbqrwOys,4529
|
|
6
6
|
tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
|
|
7
7
|
tuieval/client.py,sha256=5wo0ySMD3o8up3cxKZicUWYFvOLZB60kYBV7yxP6LjI,12378
|
|
8
|
-
tuieval/compare.py,sha256=
|
|
8
|
+
tuieval/compare.py,sha256=L5mmNY7ZUSOVG9p-_2vSZ9227TVguvIm6ZZ0n56vds8,37280
|
|
9
9
|
tuieval/dates.py,sha256=CVazlwNjCRhD__D12zXLdeqF2ySt15X3xLUQeE-o6PY,6293
|
|
10
10
|
tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
|
|
11
|
-
tuieval/engine.py,sha256=
|
|
11
|
+
tuieval/engine.py,sha256=lbdESO0J1kiZUCJ4Rl6s54wNXGJq5LuDwgm7sKR-fGc,136497
|
|
12
12
|
tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
|
|
13
|
-
tuieval/insights.py,sha256=
|
|
13
|
+
tuieval/insights.py,sha256=BDOn9UkDUkCKFcG2YIfZcnECgIFIUHxQawPRROGvDpM,29534
|
|
14
14
|
tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
|
|
15
15
|
tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
|
|
16
16
|
tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
|
|
17
17
|
tuieval/packs.py,sha256=KOj0BkwsB2124aslGyauZonyyGqtSrCJ-WAozEntmqE,17466
|
|
18
18
|
tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
|
|
19
|
-
tuieval/pta.py,sha256=
|
|
19
|
+
tuieval/pta.py,sha256=K8KE_PUcOgoZGaJHCWkPYfVHbPvd243uG0QPuxnVKto,16949
|
|
20
20
|
tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
|
|
21
|
-
tuieval/run_evals.py,sha256=
|
|
21
|
+
tuieval/run_evals.py,sha256=afZIzSdZPSCWnhc9qV_L1TXexl7YgSF6eKlV7RimMao,50679
|
|
22
22
|
tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
|
|
23
23
|
tuieval/selftest.py,sha256=cIGSDCrCK-QISAXjKSJDB4sD-WwS32bn3Gn4Y0oGoF8,11502
|
|
24
|
-
tuieval/tui.py,sha256
|
|
24
|
+
tuieval/tui.py,sha256=-7eJASdO9OhFz2KJoXNKtqeTL9DlsyafZa3mPI-tjfs,204533
|
|
25
25
|
tuieval/tune.py,sha256=izz1Qs-d9PrWAqEZpyrLxfrlhlaouif9uFQSeJveL0k,41294
|
|
26
26
|
tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
|
|
27
27
|
tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
|
|
@@ -50,8 +50,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
|
|
|
50
50
|
tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
|
|
51
51
|
tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
|
|
52
52
|
tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
|
|
53
|
-
tuieval-0.2.0.
|
|
54
|
-
tuieval-0.2.0.
|
|
55
|
-
tuieval-0.2.0.
|
|
56
|
-
tuieval-0.2.0.
|
|
57
|
-
tuieval-0.2.0.
|
|
53
|
+
tuieval-0.2.0.dev133.dist-info/METADATA,sha256=dLL6ycED3j35icT3OH8DRUbgMfLBpIUXtQPwMz_z7dI,15105
|
|
54
|
+
tuieval-0.2.0.dev133.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
55
|
+
tuieval-0.2.0.dev133.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
|
|
56
|
+
tuieval-0.2.0.dev133.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
|
|
57
|
+
tuieval-0.2.0.dev133.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|