tuieval 0.2.0.dev55__py3-none-any.whl → 0.2.0.dev62__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev55'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev55')
+__version__ = version = '0.2.0.dev62'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev62')
 
 __commit_id__ = commit_id = None

tuieval/cli.py

@@ -14,7 +14,7 @@ Start here
 
 Day to day
   tuieval compare [...]       scorecards (--speed, --pairwise, --failures)
-  tuieval pta                 the PTA index: parsimony, speed and accuracy per model, as bars
+  tuieval pta                 the PTA index: parsimony (tokens), time and accuracy per model, as bars
   tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
   tuieval items               tests that don't separate models or look broken

tuieval/compare.py

@@ -121,6 +121,12 @@ def p90(values):
     return statistics.quantiles(values, n=10)[-1] if len(values) > 1 else (values[0] if values else None)
 
 
+def pct(fraction):
+    """A share as a percentage: "100%" and "0%" only when exact, else one decimal ("99.6%"), so a model
+    with a wrong answer never shows as perfect."""
+    return "100%" if fraction >= 1 else "0%" if fraction <= 0 else f"{100 * fraction:.1f}%"
+
+
 def median(values):
     values = [v for v in values if v is not None]
     return statistics.median(values) if values else None
@@ -154,10 +160,19 @@ def settings_notes(infos):
         per_model.setdefault(i["label"], i["settings"])
     keys = sorted({k for s in per_model.values() for k in s})
     differing = [k for k in keys if len({json.dumps(s.get(k)) for s in per_model.values()}) > 1]
-    if differing:
-        detail = "; ".join(f"{k}: " + ", ".join(f"{m}={per_model[m].get(k)}" for m in sorted(per_model))
-                           for k in differing)
-        notes.append(f"Settings differ between models ({detail}). Compare those models knowing that.")
+    parts = []
+    for k in differing:   # only the models that differ from the most common value, named when few
+        groups = collections.defaultdict(list)
+        for m in sorted(per_model):
+            groups[json.dumps(per_model[m].get(k))].append(m)
+        common = max(groups, key=lambda v: len(groups[v]))
+        for value, ms in sorted(groups.items()):
+            if value != common:
+                who = ", ".join(ms) if len(ms) <= 3 else f"{len(ms)} models"
+                shown = "unset" if value == "null" else json.loads(value)
+                parts.append(f"{k} {shown} on {who}")
+    if parts:
+        notes.append("Settings differ: " + "; ".join(parts) + ".")
     fps = collections.defaultdict(set)
     for i in infos:
         if i.get("pack"):
@@ -412,7 +427,7 @@ def by_difficulty(rows):
             cells = []
             for lvl in LEVELS:
                 x = [r["ok"] for r in sub if r.get("difficulty") == lvl]
-                cells.append(f"{sum(x)}/{len(x)} ({100 * sum(x) / len(x):.0f}%)" if x else "-")
+                cells.append(f"{sum(x)}/{len(x)} ({pct(sum(x) / len(x))})" if x else "-")
             table.append([m, s] + cells)
     return header, table
 

tuieval/engine.py

@@ -987,16 +987,16 @@ class Engine:
                     del per_pack[name]
         return {l: p for l, p in out.items() if p}
 
-    def estimate_seconds(self, jobs):
-        """(seconds, notes): expected run time for the waiting jobs, estimated per pack.
+    def seconds_per_answer(self, jobs, waiting_only=True):
+        """({job key: expected seconds per answer}, borrowed, guessed, thin) for the waiting jobs.
         A pack the model has answered uses its own median time there. Otherwise the median of
         other models' times on that pack, scaled by how much slower or faster this model is than
-        them on packs both have answered. notes lists what made the estimate rough ([] = solid)."""
+        them on packs both have answered. The sets say which estimates are rough (see estimate_seconds)."""
         times = self.answer_times()
         med = {l: {p: statistics.median(ts) for p, ts in pp.items()} for l, pp in times.items()}
-        total, borrowed, guessed, thin = 0.0, set(), set(), set()
+        spr_of, borrowed, guessed, thin = {}, set(), set(), set()
         for j in jobs:
-            if j.status != "waiting":
+            if j.status in ("done", "skipped", "failed") or (waiting_only and j.status != "waiting"):
                 continue
             own = med.get(j.label, {})
             ref = {p: statistics.median(v[p] for l, v in med.items() if p in v and l != j.label)
@@ -1013,8 +1013,18 @@ class Engine:
             else:
                 spr = statistics.median(own.values()) if own else DEFAULT_SECONDS_PER_REQUEST
                 guessed.add((j.key, bool(own)))
-            done = len(self._done_keys(j)) if j.merge else 0
-            total += max(0, j.total - done) * spr
+            spr_of[j.key] = spr
+        return spr_of, borrowed, guessed, thin
+
+    def estimate_seconds(self, jobs):
+        """(seconds, notes): expected run time for the waiting jobs, estimated per pack
+        (seconds_per_answer). notes lists what made the estimate rough ([] = solid)."""
+        spr_of, borrowed, guessed, thin = self.seconds_per_answer(jobs)
+        total = 0.0
+        for j in jobs:
+            if j.key in spr_of:
+                done = len(self._done_keys(j)) if j.merge else 0
+                total += max(0, j.total - done) * spr_of[j.key]
         notes = []
         if borrowed:
             factors = sorted({f for _, f, _ in borrowed if f is not None})

tuieval/pta.py

@@ -1,13 +1,14 @@
-"""PTA index: parsimony, speed (time) and accuracy per model, 0-100 each and higher is better, shown
-as a bar per score for each model.
+"""PTA index: parsimony (tokens), time and accuracy per model, a bar each (longer = better).
 
-    P  parsimony  a score out of 100, from the tokens (reasoning and answer) used on every question
-                  compared (each question at its median over repeats): 100 for the model that used the
-                  fewest, POINTS_PER_DOUBLING points less for every doubling (0 at about 100 times more).
-                  Unlike T it doesn't depend on the machine: it's how much a model says to get there.
-    T  speed      a score out of 100, the same from the total time to answer those questions: 100 for
-                  the fastest model. Log scales, so models that are all far behind the best still differ.
-    A  accuracy   a percentage: the share of answers to those questions that passed.
+    P  parsimony (tokens)  the tokens (reasoning and answer) used on every question compared (each
+                           question at its median over repeats), shown as how many times the leanest
+                           model's ("2.4×"). Unlike T it doesn't depend on the machine.
+    T  time                the total time to answer those questions, as how many times the fastest
+                           model's ("28×").
+    A  accuracy            the share of answers to those questions that passed (%).
+
+Behind the P and T bars is a 0-100 score: 100 for the best, POINTS_PER_DOUBLING points less for every
+doubling (0 at about 100 times more), so models far behind the best still get a visible bar.
 
 Models are compared on the questions all of them answered, so one that skipped some never looks
 faster. Server and connection errors aren't answers. By default a model with fewer than half as many
@@ -32,12 +33,23 @@ def score(total, best):
     return max(0.0, 100 - POINTS_PER_DOUBLING * math.log2(total / best))
 
 
-def index(rows, models=None):
+def memory_gb(infos):
+    """{model: the most memory its server held in any run (GB)}, for models tuieval served (hosted: none).
+    On a discrete GPU that's its peak VRAM (vram_peak_gb), where the model lives; otherwise its resident memory."""
+    out = {}
+    for i in infos:
+        gb = i.get("vram_peak_gb") or (i["peak_rss_mb"] / 1024 if i.get("peak_rss_mb") else None)
+        if gb:
+            out[i["label"]] = max(out.get(i["label"], 0), gb)
+    return out
+
+
+def index(rows, models=None, memory=None):
     """{"questions": n shared, "models": [{model, P, T, A, tokens, total_s, passed, answers, alongside,
-    fewest, fastest}],
+    fewest, fastest, tps, ttft, memory_gb}],
     "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
     most-answered model}, best accuracy first. models: the ones to compare (default: every model in rows
-    with enough answers; models named here are compared whatever their count)."""
+    with enough answers; models named here are compared whatever their count). memory: memory_gb()."""
     no_answers = sorted({r["model"] for r in rows} - {r["model"] for r in rows if not r.get("error")})
     rows = [r for r in rows if not r.get("error")]     # a server or connection error isn't an answer
     tests = compare.per_question(rows)
@@ -61,12 +73,19 @@ def index(rows, models=None):
                     "total_s": sum(c["secs"] or 0 for c in cells),
                     "passed": sum(c["passed"] for c in cells), "answers": answers,
                     "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
+                    # generation speed and time to first token over the same answers (medians)
+                    "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
+                    "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
+                    "memory_gb": (memory or {}).get(m),
                     # answers that ran while other models were served shared the machine's speed
                     "alongside": any(r.get("alongside") for c in cells for r in c["rows"])})
     fastest = min((x["total_s"] for x in out if x["total_s"] > 0), default=None)
     fewest = min((x["tokens"] for x in out if x["tokens"] > 0), default=None)
     for x in out:
         x["P"], x["T"] = score(x["tokens"], fewest), score(x["total_s"], fastest)
+        # shown on the bars: how many times the leanest / fastest model's total (1.0 = the best)
+        x["tokens_x"] = x["tokens"] / fewest if fewest and x["tokens"] else None
+        x["time_x"] = x["total_s"] / fastest if fastest and x["total_s"] else None
         x["fewest"] = bool(fewest) and x["tokens"] == fewest
         x["fastest"] = bool(fastest) and x["total_s"] == fastest
     out.sort(key=lambda x: (-(x["A"] or 0), -(x["T"] or 0), x["model"]))
@@ -74,29 +93,40 @@ def index(rows, models=None):
             "most": most}
 
 
-LETTERS = (("P", "parsimony", "/100", "cyan"), ("T", "speed", "/100", "magenta"), ("A", "accuracy", "%", "green"))
+# (letter, name, what the number beside the bar shows, colour); the bar itself is the 0-100 score
+LETTERS = (("P", "parsimony (tokens)", "tokens_x", "cyan"), ("T", "time", "time_x", "magenta"),
+           ("A", "accuracy", "A", "green"))
+
+
+def times(v):
+    """A multiple of the best: "1.0×", "2.4×", "28×"."""
+    return "-" if v is None else f"{v:.1f}×" if v < 9.95 else f"{v:.0f}×"
 
 
 def bars(result, width=20, markup=True):
-    """Lines with a row per model and a bar per score (P, T, A), longer = better, each with its number:
-    the at-a-glance view. markup=False gives plain text (the report)."""
+    """Lines with a row per model and a bar per score (P, T, A), longer = better: the at-a-glance view.
+    Beside P and T, how many times the leanest / fastest model's tokens / time; beside A, the percentage.
+    markup=False gives plain text (the report)."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
-    cell_w = width + 6                          # the bar, a space and "100%" or " 100"
+    # each column: the bar, a space, "100%" or "9.8×" (5), then a gap; at least as wide as its heading
+    cell_w = max(width + 9, *(len(f"{k} {name}") + 2 for k, name, _, _ in LETTERS))
     head = " " * (name_w + 2) + "".join(f"{f'{k} {name}':<{cell_w}}" for k, name, _, _ in LETTERS)
     lines = [f"[b]{head.rstrip()}[/b]" if markup else head.rstrip()]
     for x in result["models"]:
         cells = []
-        for k, _, unit, color in LETTERS:
+        for k, _, shown, color in LETTERS:
             v = x[k]
             full = 0 if v is None else round(width * max(0, min(100, v)) / 100)
+            if v is not None and v < 100:   # full only when perfect (accuracy) or the best (P, T)
+                full = min(full, width - 1)
             bar, rest = "█" * full, "░" * (width - full)
-            num = "-" if v is None else f"{v:.0f}{'%' if unit == '%' else ''}"
+            num = (("-" if v is None else compare.pct(v / 100)) if shown == "A" else times(x.get(shown)))
             bar = f"[{color}]{bar}[/][grey37]{rest}[/]" if markup else bar + rest
-            cells.append(f"{bar} {num:>4}")
+            cells.append(f"{bar} {num:>6}" + " " * (cell_w - width - 7))
         name = f"{x['model']:<{name_w}}"
-        lines.append(f"{rich_escape(name) if markup else name}  " + "  ".join(cells))
+        lines.append((f"{rich_escape(name) if markup else name}  " + "".join(cells)).rstrip())
     return lines
 
 
@@ -116,22 +146,28 @@ def left_out_lines(result):
     return out
 
 
-def _score(v, unit=""):
-    """A is a percentage (%), P and T scores out of 100 (/100: on a log scale, not percentages)."""
-    return "-" if v is None else f"{v:.0f}{unit}"
-
-
 def table(result):
     """(header, rows) for a plain table of the index."""
-    header = ["model", "P parsimony /100", "T speed /100", "A accuracy %", "tokens", "total time",
-              "answers right"]
-    rows = [[x["model"], _score(x["P"]), _score(x["T"]), _score(x["A"], "%"),
-             f"{round(x['tokens']):,}" + (" ★" if x.get("fewest") else ""),
-             fmt_total(x["total_s"]) + (" ★" if x.get("fastest") else ""),
-             f"{x['passed']}/{x['answers']}"] for x in result["models"]]
+    n = result["questions"] or 1
+    header = ["model", "tokens/answer", "time/answer", "answers right", "tok/s", "TTFT s", "memory GB"]
+    rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
+             fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
+             f"{x['passed']}/{x['answers']}", _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
+             _num(x.get("memory_gb"), ".1f")] for x in result["models"]]
     return header, rows
 
 
+def fmt_secs(secs):
+    """Time per answer: "2.5s", "69s", "4m10s"."""
+    if not secs:
+        return "-"
+    return f"{secs:.1f}s" if secs < 10 else f"{secs:.0f}s" if secs < 60 else fmt_total(secs)
+
+
+def _num(v, fmt):
+    return "-" if v is None else format(v, fmt)
+
+
 def fmt_total(secs):
     if not secs:
         return "-"
@@ -147,19 +183,18 @@ def scope_note(result, models):
     if not result["questions"]:
         return "These models have no question in common yet, so there's nothing to compare."
     return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
-            f"A is a percentage. P parsimony and T speed are scores out of 100, not percentages: 100 = the fewest "
-            f"tokens / the fastest total time (★), {POINTS_PER_DOUBLING} points less for each doubling; a better "
-            "model added lowers the others.")
+            "Longer bars are better. P parsimony (tokens) and T time: how many times the leanest / fastest "
+            "model's (1.0×, ★ in the table). A accuracy: answers right.")
 
 
 def for_engine(e, labels=None, packs=None, results_dir=None):
     """(result, models) from a workspace's finished results (results on their way out excluded)."""
     paths = [p for p in compare.default_paths(results_dir or e.results_dir) if not e.superseded(p)]
-    rows, _ = compare.load(paths) if paths else ([], [])
+    rows, infos = compare.load(paths) if paths else ([], [])
     if packs:
         rows = [r for r in rows if r["suite"] in packs]
     models = sorted({r["model"] for r in rows} & set(labels) if labels else {r["model"] for r in rows})
-    return index(rows, models if labels else None), models
+    return index(rows, models if labels else None, memory_gb(infos)), models
 
 
 def markdown(result, models):

tuieval/run_evals.py

@@ -373,11 +373,12 @@ def report_markdown(e, labels=None):
                              + f"; truncated {100 * ev['truncation']:.1f}%"
                              + (f"; consistency {100 * ev['consistency']:.0f}% of {ev['groups']} groups" if ev.get("consistency") is not None else ""))
                 recs = e.records(label, name)
+                from .compare import pct
                 levels = []
                 for lvl in ("easy", "medium", "hard"):
                     x = [r["pass"] for r in recs if r.get("difficulty") == lvl]
                     if x:
-                        levels.append(f"{lvl} {sum(x)}/{len(x)} ({100 * sum(x) / len(x):.0f}%)")
+                        levels.append(f"{lvl} {sum(x)}/{len(x)} ({pct(sum(x) / len(x))})")
                 if levels:
                     lines.append("  - by difficulty: " + ", ".join(levels))
                 for ex in ev.get("critical_examples", []):
@@ -389,7 +390,7 @@ def report_markdown(e, labels=None):
 
 def cmd_pta(argv):
     p = argparse.ArgumentParser(prog="tuieval pta",
-                                description="The PTA index: parsimony (tokens), speed and accuracy per model (0-100, higher is better), as a "
+                                description="The PTA index: parsimony (tokens), time and accuracy per model, as a "
                                             "bar per score for each model and a table, over the questions all the models answered.")
     p.add_argument("--only", help="comma-separated model labels (default: every model with results)")
     p.add_argument("--packs", help="comma-separated packs (default: all)")
@@ -413,7 +414,7 @@ def cmd_pta(argv):
         return
     out.print()
     for line in pta.bars(result, a.width):
-        out.print(line)
+        out.print(line, soft_wrap=True)   # a bar row stays one line (the terminal wraps it if it must)
     header, rows = pta.table(result)
     widths = [max(len(h), *(len(r[i]) for r in rows)) for i, h in enumerate(header)]
     out.print()

tuieval/tui.py

@@ -10,7 +10,7 @@ Run screen:    live reasoning and answer, per-test ✓/✗, TTFT and tokens/s, p
                k skips the current model, c cancels everything, f pauses auto-scroll.
                r shows results at any time (also mid-run); n starts a new run when it ends.
                Enter on a Recent results row shows that answer in full.
-Results:       opens on the PTA index (parsimony, speed and accuracy per model, as bars). Per
+Results:       opens on the PTA index (parsimony, time and accuracy per model, as bars). Per
                question compares each model's tokens and seconds on the same question. ctrl+r
                reloads (it also reloads when a run's pack finishes). Enter on a Failures row or a
                Per question cell shows the answer: grading and checks, reasoning, answer, question.
@@ -19,6 +19,7 @@ Anywhere:      esc/q go back (q quits only on Setup), ctrl+q quits, w lists this
                tunes. Leaving a live run or tune keeps it going; the header shows its progress.
 """
 import argparse
+import collections
 import glob
 import json
 import os
@@ -331,16 +332,15 @@ with the model's reasoning and answer as they stream, then recent results with t
   v  with several models at a time: stream the next one (k skips the one streaming)""",
     "ResultsScreen": """[b]Results[/b]
 
-Opens on the [b]PTA index[/b]: parsimony (tokens), speed and accuracy per model, a bar each (0-100,
-longer = better). [b]Production readiness[/b] gives one verdict per model and use case, and what's
-missing when it's INCONCLUSIVE. The other tabs are the evidence:
-  Scorecard              accuracy, critical failures and truncation per model and pack
-  Speed & tokens         time and tokens per answer (★ = nothing beats it on both accuracy and time)
-  Per question           every question, model by model
-  Is the difference real? whether one model is really better than another, or it's noise
-  Tests that separate    the questions that tell models apart
+  PTA index              parsimony (tokens), time and accuracy per model, a bar each (longer = better;
+                         2.4× = 2.4 times the best), with per-answer numbers, tok/s, TTFT and memory in
+                         the table. Results open here.
+  Production readiness   one verdict per model and use case; the table says why and what to run next
+  Per question           every question, model by model (filters: pack, level, Models disagree)
   Failures               every wrong answer with the grader's reason; enter opens it in full
-  Test quality           tests nobody fails, everybody fails, or that look broken
+
+More in the terminal: tuieval compare (--pairwise: is a difference real), tuieval history,
+tuieval items (test quality).
 
 [b]Keys[/b]  esc back · ctrl+r refresh · w runs""",
     "TuneScreen": """[b]Tuning speed flags[/b]
@@ -1855,6 +1855,8 @@ class RunScreen(Screen):
         self.current = ""
         self.req_first = None
         self.stats = []
+        self.spr, self.rough = None, set()   # job key -> expected seconds per answer (ETA), rough ones
+        self.live = {}                        # job key -> seconds per answer in this session
         self.mem = {}
         self.outcome = ""  # "Finished" or "Cancelled" once the queue is done
         self.ended = None
@@ -2037,6 +2039,8 @@ class RunScreen(Screen):
                            + (f"  [red]copy of repeat {r['identical_to_repeat'] + 1}[/red]"
                               if r.get("identical_to_repeat") is not None else ""))
             self.stats.append(r)
+            if r.get("total_s"):
+                self.live.setdefault(job.key, []).append(r["total_s"])
             self.recent_entries[row_key] = {"model": job.label, "pack": job.pack.name, "record": r,
                                             "pack_fp": job.pack.fingerprint}
             if recent.row_count > 500:
@@ -2187,6 +2191,34 @@ class RunScreen(Screen):
                            ("note", short_note(j.note))):
             q.update_cell(j.key, col, value, update_width=True)
 
+    LIVE_ANSWERS = 5   # a pack's own times this session take over from its history after this many
+
+    def eta_seconds(self):
+        """(seconds, rough): time left, pack by pack: answers left × seconds per answer. A pack's own
+        average this session once it has LIVE_ANSWERS of them, before that its history (Engine.
+        seconds_per_answer: this model's past times there, or other models' scaled). Divided by the
+        models running side by side. rough: some packs have no history of their own yet."""
+        new = [j for j in self.jobs if self.spr is None or (j.key not in self.spr and j.status not in
+                                                           ("done", "skipped", "failed"))]
+        if new:   # at the start, and for evals added to the run
+            spr, borrowed, guessed, _ = self.app.engine.seconds_per_answer(new, waiting_only=False)
+            self.spr = {**(self.spr or {}), **spr}
+            self.rough |= {k for k, *_ in borrowed} | {k for k, _ in guessed}
+        left, rough, models = 0.0, False, set()
+        for j in self.jobs:
+            if j.status in ("done", "skipped", "failed") or j.done >= j.total:
+                continue
+            live = self.live.get(j.key, [])
+            if len(live) >= self.LIVE_ANSWERS:
+                spr = sum(live) / len(live)
+            else:
+                spr = self.spr.get(j.key, engine.DEFAULT_SECONDS_PER_REQUEST)
+                rough |= j.key in self.rough
+            left += (j.total - j.done) * spr
+            models.add(j.label)
+        lanes = max(1, min(self.parallel or self.app.engine.parallel_models(), len(models)))
+        return left / lanes, rough
+
     def update_overall(self):
         done = sum(j.done for j in self.jobs)
         passed = sum(j.passed for j in self.jobs)
@@ -2194,10 +2226,10 @@ class RunScreen(Screen):
         self.query_one("#bar", ProgressBar).update(progress=done)
         elapsed = time.time() - self.started
         eta = ""
-        done_now = sum(j.done - j.resumed for j in self.jobs if j.started)   # answers from earlier sittings
-        if self.running and done_now > 0 and self.total > done:              # took no time in this one
-            eta = f" · ETA {fmt_secs(elapsed / done_now * (self.total - done))}"
-        pct = f"{100 * passed / (passed + failed):.0f}%" if passed + failed else "-"
+        if self.running and self.total > done:
+            left, rough = self.eta_seconds()
+            eta = f" · ETA {'~' if rough else ''}{fmt_secs(left)}"
+        pct = compare.pct(passed / (passed + failed)) if passed + failed else "-"
         recent = self.stats[-50:]
         tps = [r["gen_tps"] for r in recent if r.get("gen_tps")]
         ttfts = [r["ttft_s"] for r in recent if r.get("ttft_s") is not None and not r.get("cached_tokens")]
@@ -2315,8 +2347,8 @@ class RunScreen(Screen):
 class ResultsScreen(Screen):
     BINDINGS = [Binding("escape", "app.pop_screen", "Back"), Binding("q", "app.pop_screen", "Back", show=False),
                 Binding("w", "runs", "Runs"), Binding("ctrl+r", "refresh", "Refresh")]
-    SORTS = [("Sort: token gap", "tokens"), ("Sort: time gap", "secs"),
-             ("Sort: most tokens", "most"), ("Sort: pack order", "order")]
+    SORTS = [("By token gap", "tokens"), ("By time gap", "secs"),
+             ("By most tokens", "most"), ("By pack order", "order")]
 
     def check_action(self, action, parameters):
         return bool(self.app.sessions) if action == "runs" else True
@@ -2346,9 +2378,9 @@ class ResultsScreen(Screen):
                 with Horizontal(id="pta-controls"):
                     yield Select([("All packs", "")], id="pta-pack", allow_blank=False, value="")
                     yield Button("Models: all", id="pta-models")
-                yield Static("[dim]P parsimony: tokens used (reasoning and answer). T speed: total time. A accuracy: "
-                             "answers right. Longer bars are better for all three. Verdicts and critical failures "
-                             "are in Production readiness.[/dim]")
+                yield Static("[dim]P parsimony (tokens): tokens used, reasoning and answer. T time: total time. "
+                             "A accuracy: answers right. Verdicts and critical failures are in Production "
+                             "readiness.[/dim]")
                 yield Static(id="pta-note")
                 yield Static(id="pta-bars")
                 yield DataTable(id="pta", zebra_stripes=True, cursor_type="none")
@@ -2361,22 +2393,6 @@ class ResultsScreen(Screen):
                              "measured there, or projected from token counts and that machine's tuned speeds.[/dim]")
                 yield DataTable(id="latency", zebra_stripes=True)
                 yield DataTable(id="readiness-detail", zebra_stripes=True, cursor_type="row")
-            with TabPane("Verdict history"):
-                yield Static("[dim]Every change in a model's verdict, newest first. Kept even after results are rerun "
-                             "or questions change; the evidence behind older verdicts is in results/<model>/history/.[/dim]")
-                yield DataTable(id="history", zebra_stripes=True, cursor_type="row")
-            with TabPane("Scorecard"):
-                yield DataTable(id="scorecard", zebra_stripes=True)
-            with TabPane("Speed & tokens"):
-                yield Select([("As measured (all machines)", "")] +
-                             [(f"{mc.id}{' (this machine)' if i == 0 else ''}", mc.id)
-                              for i, mc in enumerate(self.app.engine.machines())],
-                             id="speed-machine", allow_blank=False, value="")
-                yield Static("[dim]★ = no other model is both more accurate and faster per answer. "
-                             "Memory is the server's resident size (includes GPU-mapped weights on Apple Silicon). "
-                             "On another machine, times are measured there if the model ran there, otherwise "
-                             "projected from each answer's token counts and that machine's tuned speeds.[/dim]")
-                yield DataTable(id="speed", zebra_stripes=True)
             with TabPane("Per question", id="tab-per-question"):
                 with Horizontal(id="pq-controls"):
                     yield Select([("All packs", "")], id="pq-pack", allow_blank=False, value="")
@@ -2384,28 +2400,16 @@ class ResultsScreen(Screen):
                                  id="pq-level", allow_blank=False, value="")
                     yield Select(self.SORTS, id="pq-sort", allow_blank=False, value="tokens")
                     yield Button("Models: all", id="pq-models")
-                    yield Checkbox("Only questions all answered", False, id="pq-shared")
-                    yield Checkbox("Include unfinished", True, id="pq-partial")
+                    yield Checkbox("Answered by all", False, id="pq-shared")
+                    yield Checkbox("Models disagree", False, id="pq-disagree")
+                    yield Checkbox("In progress", True, id="pq-partial")
                 yield Static(id="pq-compare-title")
                 yield DataTable(id="pq-summary", zebra_stripes=True, cursor_type="none")
                 yield Static(id="pq-legend")
                 yield DataTable(id="per-question", zebra_stripes=True, cursor_type="cell")
-            with TabPane("Is the difference real?"):
-                yield DataTable(id="pairwise", zebra_stripes=True)
-            with TabPane("Tests that separate models"):
-                yield DataTable(id="separating", zebra_stripes=True)
-            with TabPane("By difficulty"):
-                yield Static("[dim]Pass rate per difficulty level. The level is your label only; models never see it. "
-                             "A strong model should hold up on HARD, not just EASY.[/dim]")
-                yield DataTable(id="difficulty", zebra_stripes=True)
             with TabPane("Failures"):
                 yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
                 yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
-            with TabPane("Test quality"):
-                yield Static("[dim]no signal: every model passed · suspicious: every model failed (check the test) · "
-                             "inverted: the weakest model beat the strongest · flaky: same model, different results "
-                             "across repeats[/dim]")
-                yield DataTable(id="quality", zebra_stripes=True)
         yield Footer()
 
     def on_mount(self):
@@ -2462,7 +2466,6 @@ class ResultsScreen(Screen):
 
     def fill(self):
         results_dir = self.results_dir or self.app.engine.results_dir
-        self.fill_history()
         paths = compare.default_paths(results_dir)
         self.partial_rows = compare.load_partials(results_dir)
         self.superseded = {p for p in compare.default_paths(results_dir) if self.app.engine.superseded(p)}
@@ -2479,40 +2482,13 @@ class ResultsScreen(Screen):
         smoke = os.path.abspath(results_dir) != os.path.abspath(self.app.engine.results_dir)
         info.update(f"{len(paths)} {'Smoke ' if smoke else ''}result files in {os.path.relpath(results_dir)}/" + banner
                     + "".join(f"\n[yellow]⚠ {n}[/yellow]" for n in notes))
-        header, table = compare.scorecard(rows)
-        t = self.query_one("#scorecard", DataTable)
-        t.add_columns(*header)
-        t.add_rows(table)
         self.rows, self.infos = rows, infos
-        self.fill_speed(self.query_one("#speed-machine", Select).value or "")
-        p = self.query_one("#pairwise", DataTable)
-        p.add_columns("Model A", "Model B", "A − B", "95% CI", "Verdict", "Shared tests")
-        for a, b, diff, lo, hi, call, n in compare.pairwise(rows):
-            color = "green" if call == "clear" else "dim"
-            p.add_row(a, b, f"{100 * diff:+.1f} pts", f"[{100 * lo:+.1f}, {100 * hi:+.1f}]",
-                      f"[{color}]{call}[/{color}]", str(n))
-        split, _ = compare.separating(rows)
-        sep = self.query_one("#separating", DataTable)
-        models = sorted({r["model"] for r in rows})
-        sep.add_columns("Test", *models)
-        for test, d in split:
-            sep.add_row(test, *[f"{sum(d[m])}/{len(d[m])}" if m in d else "-" for m in models])
         f = self.query_one("#failures", DataTable)
         f.add_columns("Model", "Level", "Test", "Reason")
         # compare.failures lists the failed rows in order, so they line up with fail_rows
         self.fail_rows = [r for r in rows if not r["ok"]]
         for model, test, reason, level in compare.failures(rows):
             f.add_row(model, diff_badge(level), test, rich_escape(reason[:130]))
-        dh, dt = compare.by_difficulty(rows)
-        dtab = self.query_one("#difficulty", DataTable)
-        dtab.add_columns("Model", "Pack", *[diff_badge(l) for l in compare.LEVELS])
-        for line in dt:
-            dtab.add_row(*line)
-        qt = self.query_one("#quality", DataTable)
-        qt.add_columns("Flags", "Test", *models)
-        if len(models) >= 2:
-            for test, flags, rates in compare.items(rows):
-                qt.add_row(", ".join(flags), test, *[f"{100 * rates[m]:.0f}%" if m in rates else "-" for m in models])
         self.fill_per_question()
         self.fill_pta()
         self.fill_readiness(results_dir)
@@ -2533,6 +2509,7 @@ class ResultsScreen(Screen):
     @on(Select.Changed, "#pq-sort")
     @on(Checkbox.Changed, "#pq-partial")
     @on(Checkbox.Changed, "#pq-shared")
+    @on(Checkbox.Changed, "#pq-disagree")
     def per_question_changed(self):
         if getattr(self, "rows", None) is not None:
             self.fill_per_question()
@@ -2592,7 +2569,7 @@ class ResultsScreen(Screen):
             self.query_one("#pta-left-out", Static).update("")
             return
         # none picked: every model with enough answers (pta.index leaves out the rest, and says so)
-        result = pta.index(rows, sorted(chosen) or None)
+        result = pta.index(rows, sorted(chosen) or None, pta.memory_gb(self.infos))
         self.query_one("#pta-note", Static).update(pta.scope_note(result, sorted(present)))
         self.query_one("#pta-bars", Static).update("\n".join(pta.bars(result)) if result["questions"] else "")
         self.query_one("#pta-left-out", Static).update(
@@ -2639,6 +2616,8 @@ class ResultsScreen(Screen):
         shown = sorted(chosen) if chosen else sorted({m for t in tests for m in t["cells"]})
         if self.query_one("#pq-shared", Checkbox).value:
             tests = [t for t in tests if all(m in t["cells"] for m in shown)]
+        if self.query_one("#pq-disagree", Checkbox).value:   # some models got it right, others didn't
+            tests = [t for t in tests if len({c["passed"] == c["n"] for c in t["cells"].values()}) > 1]
         self.fill_pq_summary(tests, shown)
         order = {}
         for name, pk in self.app.engine.packs.items():
@@ -2769,38 +2748,6 @@ class ResultsScreen(Screen):
             return
         self.app.push_screen(AnswerScreen(entries, what="repeat"))
 
-    @on(Select.Changed, "#speed-machine")
-    def machine_changed(self, event):
-        if getattr(self, "rows", None) is not None:
-            self.fill_speed(event.value or "")
-
-    def fill_speed(self, machine_id):
-        e = self.app.engine
-        speeds = {}
-        if machine_id:
-            mc = next(x for x in e.machines() if x.id == machine_id)
-            speeds = {m["label"]: (e.serving(m, mc).profile or {}).get("measured") for m in e.cfg["models"]}
-        header, table, _ = compare.speed(self.rows, self.infos, machine_id or None, speeds)
-        sp = self.query_one("#speed", DataTable)
-        sp.clear(columns=True)
-        sp.add_columns(*header)
-        for line in table:
-            sp.add_row(*([f"[yellow]{line[0]}[/yellow]"] + line[1:]))
-
-    def fill_history(self):
-        e = self.app.engine
-        t = self.query_one("#history", DataTable)
-        t.add_columns("When", "Model", "Use case", "Change", "Questions", "Why")
-        rank = {"FAIL": 0, "INCONCLUSIVE": 1, "NO DATA": 1, "PASS": 2}
-        for h in reversed(verdict.load_history(e)):
-            prev, now = h.get("previous"), h["status"]
-            change = f"[{VERDICT_STYLE[now]}]{now}[/]" if not prev else \
-                f"{prev} → [{VERDICT_STYLE[now]}]{now}[/]"
-            if prev and rank.get(now, 1) < rank.get(prev, 1):
-                change += "  [bold red]⚠ worse[/bold red]"
-            qs = "current" if verdict.is_current(h, e) else "[yellow]changed since[/yellow]"
-            t.add_row(h["time"].replace("T", " "), h["model"], h["use_case"], change, qs, "; ".join(h["reasons"])[:120])
-
     def fill_readiness(self, results_dir):
         style = {"PASS": "bold green", "FAIL": "bold red", "INCONCLUSIVE": "yellow", "NO DATA": "dim"}
         matrix = self.query_one("#readiness", DataTable)
@@ -2812,17 +2759,18 @@ class ResultsScreen(Screen):
             return
         help_text.update(self.READINESS_HELP)
         table = verdict.readiness(self.app.engine)
-        short = ["[b]In short[/b]"]
+        short = ["[b]In short[/b]"]   # one line per model; what to run next is in the table below
         for label, sentence, todo in verdict.plain_summary(table):
             short.append(f"  [b]{rich_escape(label)}[/b]: {rich_escape(sentence)}")
-            short += [f"     → {rich_escape(g)}: {rich_escape(what)}" for g, what, _ in todo]
         if any("Certify" in what for _, _, todo in verdict.plain_summary(table) for _, what, _ in todo):
             short.append("  [dim]To run Certify: in Setup tick the model and pack, choose Certify, press s.[/dim]")
         self.query_one("#readiness-short", Static).update("\n".join(short) if table else
                                                           "[dim]No verdicts yet: run something from Setup.[/dim]")
         groups = sorted({g for t in table.values() for g in t})
         matrix.add_columns("Model", *groups)
-        detail.add_columns("Model", "Use case", "Pack", "Verdict", "Why")
+        detail.add_columns("Model", "Use case", "Pack", "Verdict", "Answers right", "Why")
+        right = collections.Counter((r["model"], r["suite"]) for r in self.rows if r["ok"])
+        answered = collections.Counter((r["model"], r["suite"]) for r in self.rows)
         lat = verdict.latency_table(self.app.engine, None, table)
         machine_ids = [mc.id for mc in self.app.engine.machines()]
         lt = self.query_one("#latency", DataTable)
@@ -2839,7 +2787,9 @@ class ResultsScreen(Screen):
             for g, (_, packs) in gs.items():
                 for name, pv in packs.items():
                     if pv.status != "NO DATA":
-                        detail.add_row(label, g, name, f"[{style[pv.status]}]{pv.status}[/]", "; ".join(pv.reasons)[:160])
+                        n = answered[(label, name)]
+                        detail.add_row(label, g, name, f"[{style[pv.status]}]{pv.status}[/]",
+                                       f"{right[(label, name)]}/{n}" if n else "-", "; ".join(pv.reasons)[:160])
 
 
 # ------------------------------------------------------------------ app
@@ -2900,7 +2850,6 @@ class EvalsApp(App):
     #machine { padding: 0 1; height: auto; }
     #tune-status { padding: 0 1; height: auto; min-height: 2; background: $boost; }
     #tune-log { height: 1fr; }
-    #speed-machine { width: 60; margin: 0 1; }
     #results-top { height: auto; }
     #results-info { width: 1fr; height: auto; }
     #toggle-smoke { width: auto; min-width: 32; }
@@ -2911,7 +2860,7 @@ class EvalsApp(App):
     #pta { width: 1fr; height: auto; margin-top: 1; }
     #pta-left-out { height: auto; padding: 1 0 0 0; }
     #pq-controls Checkbox { width: auto; }
-    #pq-controls Select { width: 28; margin-right: 1; }
+    #pq-controls Select { width: 22; margin-right: 1; }
     #pq-models { margin-right: 1; min-width: 20; }
     #pq-compare-title { padding: 0 1; height: auto; }
     #pq-summary { height: auto; max-height: 10; }

tuieval/verdict.py

@@ -259,7 +259,8 @@ def _plain_fail(v):
     if "cut off" in text:
         return "too many answers cut off by max_tokens"
     if "accuracy" in text and ev.get("trials"):
-        return f"{100 * ev['accuracy']:.0f}% right, under the bar"
+        from .compare import pct   # local: compare imports nothing from here, but keep verdict light
+        return f"{pct(ev['accuracy'])} right, under the bar"
     return text.split(" (")[0]
 
 

{tuieval-0.2.0.dev55.dist-info → tuieval-0.2.0.dev62.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev55
+Version: 0.2.0.dev62
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -144,7 +144,7 @@ See **[docs/models.md](docs/models.md)** for servers, OpenRouter endpoint pinnin
 1. **Pick packs and models** (space ticks; type to filter models by label or tag). Each model shows a short code per use case, e.g. `C✓ S?` (✓ pass, ✗ fail, ? inconclusive; grey = from earlier results). To run only some tests of a pack, highlight it and press `e`. Runs of different tests add up, and the pack can PASS once all its tests have run.
 2. **Pick a tier** (Smoke to check setup, Screen, Certify) and **press `s`**. The line above the buttons shows how many answers that is and roughly how long it will take. For each model, tuieval starts its server (or uses a running one), checks the right model is loaded, runs every selected pack, then stops it.
 3. **Watch the run:** progress with ETA, live score, tok/s, TTFT and memory, the current test with reasoning and answer side by side, recent results with the grader's reason. `k` skips a model, `c` cancels (finished work is kept and resumes next time). `n` sets up more evals while it runs: add them to this run (same tier) or queue them as the next run.
-4. **Press `r` for results:** production readiness, verdict history, scorecard, speed & tokens (★ marks models nothing beats on both accuracy and time), per question, is the difference real?, tests that separate models, by difficulty, failures, and test quality.
+4. **Press `r` for results:** the PTA index (parsimony (tokens), time and accuracy as bars), production readiness, per question, and failures. `tuieval compare`, `tuieval history` and `tuieval items` have the rest: whether a difference is real, verdict history and test quality.
 
 Other keys: `t` tunes the ticked models' speed flags, `a` adds a model, `m` scans for GGUFs, `p` saves or loads a preset, `x` hides a model. `?` explains any screen.
 
@@ -176,7 +176,7 @@ tuieval help
 
 - **Start with Production readiness.** A single critical failure already means FAIL, whatever the accuracy: a model that breaks a hard rule once in 300 answers will do it in production. The Failures tab and `tuieval report` list exactly which answers failed.
 - **INCONCLUSIVE is not "nearly passed".** It says what's missing (usually a Certify run, or more trials).
-- **Trust "Is the difference real?" over raw percentages.** Differences of one or two tests are usually noise.
+- **Trust `tuieval compare --pairwise` over raw percentages.** Differences of one or two tests are usually noise.
 - **Read `trunc` before accuracy.** A model that runs out of tokens isn't wrong; it's thinking too long for the budget.
 - **Watch TTFT for interactive use.** A model that is 5% more accurate but takes 3 s longer to start answering may be the worse choice.
 

{tuieval-0.2.0.dev55.dist-info → tuieval-0.2.0.dev62.dist-info}/RECORD

@@ -1,25 +1,25 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=1ZPMt5NERZZN86PFadmP7kAdrSCtNEk8dYfg9p2vGXg,535
+tuieval/_version.py,sha256=Q5t2lKtGGZuhOAPSq1URLDDN-gL2EG-f55AiFxUzLv0,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
-tuieval/cli.py,sha256=gzGBzhqvR0WKJQQNHB8CudpgViIeIyawrNWa54whmoI,4502
+tuieval/cli.py,sha256=fn6z0cupM-aW3fY5fFgnS-HmOQmMzlDg-U76Y6LgL3o,4510
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
-tuieval/compare.py,sha256=SHN_o_nlppFcHMBqBZRITIkeQDPaR_JxhUZItQLMBwo,26270
+tuieval/compare.py,sha256=V_K34LFnDvd7xhGDS4zZDCguaIMU05f5mCbPx0VpUOA,26944
 tuieval/doctor.py,sha256=y1C_ggjd8oikXTrovgTBf26wjThP8EoVkfTVeO_n5dQ,6070
-tuieval/engine.py,sha256=w3KsgxzFAMcRyVzFGQXhTgN3d9qYRtAZZxNLXZOgnrQ,114222
+tuieval/engine.py,sha256=0PuHFus1k02qAY8nlsI6z5iQo7IT82bMwmHDZT6fGtU,114784
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/machines.py,sha256=tdRrn3WNOmCHwfcxcWz1NuAuOJdI7WnJFR1y6xgF89c,13512
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=tiLmxsrX_M5EUMuR4Yvb7pLBLEI15wamHSjflD6wjoY,11299
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
-tuieval/pta.py,sha256=KkrOxE1hTDwu1435IrzMzIysrgVUvFuKVo786VlRyK4,9056
+tuieval/pta.py,sha256=6HwPMvTTJFoZjO-2ZurucJrp1itIzpDTKL1xvYP9W0w,10888
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=5bQEqU533XYiLze9dYcl72c5GSIpohMj1eqJrePRwnc,46450
+tuieval/run_evals.py,sha256=1zNkCdFt744cqpebqiyj-sZrbOI22c6IYBcXk9koz20,46538
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=21Re8fM5rf3zbHIKvK9laMV-mL2HjcGfMReoRg0DRLw,164268
+tuieval/tui.py,sha256=lrIv4z21s9OG809706Z-lE2-TZ6gg8z0AjaJ_zW7zkw,161021
 tuieval/tune.py,sha256=jXwYuSjfdqbLCE3WonP2xaUHwk_FnhI67twB1BaHggI,38992
-tuieval/verdict.py,sha256=Muo1a05mCv781MlEJsXwDsGaKbRi0JS4Lv8HdvK_Aco,19054
+tuieval/verdict.py,sha256=VB-0LTxR9y8bJHT9RicxtE9K7cUHBdCu40hOCrsbmA4,19150
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
 tuieval/yamlout.py,sha256=r1IokjLqW-Ztpg1fSn33ZW3VK00v1VvxymmU5Qy7GEw,606
@@ -46,8 +46,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev55.dist-info/METADATA,sha256=DGIO8VwyZC4CMfIgLwiY7gpg24S1xfuPDakBj_W90qo,13881
-tuieval-0.2.0.dev55.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev55.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev55.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev55.dist-info/RECORD,,
+tuieval-0.2.0.dev62.dist-info/METADATA,sha256=X2v6zWEn3E0Lkm-W2x-pjAHgG6qjjzFw_BqeSvyyIxI,13898
+tuieval-0.2.0.dev62.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev62.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev62.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev62.dist-info/RECORD,,