tuieval 0.2.0.dev92__py3-none-any.whl → 0.2.0.dev96__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev92'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev92')
+__version__ = version = '0.2.0.dev96'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev96')
 
 __commit_id__ = commit_id = None

tuieval/cli.py

@@ -15,6 +15,7 @@ Start here
 Day to day
   tuieval compare [...]       scorecards (--speed, --pairwise, --reference M, --failures)
   tuieval pta                 the PTA index: parsimony (tokens), time and accuracy per model, as bars
+  tuieval insights            charts with what to do next: vs a reference, per use case, why models fail, …
   tuieval report              the verdicts with their evidence, as a markdown file
   tuieval history             every verdict change, newest first
   tuieval items               tests that don't separate models or look broken

tuieval/insights.py

@@ -0,0 +1,538 @@
+"""Insights: what the results say to do next, as a chart per question with plain action lines under it
+(Results → Insights, `tuieval insights`).
+
+  1 vs reference    each model against one reference model, paired on the questions both answered:
+                    accuracy points with a 95% CI, better / tie / worse
+  2 use cases       accuracy per model and use case (pack group) as a heatmap; the best marked
+  3 vs base         a fine-tune against its base model (models.toml `base`): per pack, and the question
+                    categories it gained or lost most
+  4 why they fail   each model's answers split into right, wrong, cut off, no answer / bad format, code
+                    crashed or hung, and critical failures
+  5 accuracy / cost accuracy against median tokens (or seconds) per question; the models no other beats
+                    on both
+  6 questions       questions every model fails (check them), every model passes (no signal), and the
+                    ones that split models most
+  7 next runs       the Certify runs that would settle the open verdicts of the strongest models, with
+                    their time and command
+
+Every panel returns {"title", "lines", "actions"}: lines are Rich markup (the chart), actions plain
+sentences. Server and connection errors are never answers here either.
+"""
+import collections
+import math
+import statistics
+
+from rich.markup import escape as rich_escape
+
+from . import compare
+
+MIN_SHARE = 0.5      # like the PTA index: models with fewer answers than this share of the most-answered are left out
+HALF = 15            # half width of a diverging bar
+COVERED = 0.8        # a use case counts as covered when a model answered this share of its questions
+MAX_ACTIONS = 6
+
+
+def _esc(s):
+    return rich_escape(str(s))
+
+
+def prepare(rows, picked=None, ref=None):
+    """(rows without errors, the models to show): picked ones, else every model with enough answers;
+    the reference is always shown when it has answers."""
+    rows = [r for r in rows if not r.get("error")]
+    counts = collections.Counter(r["model"] for r in rows)
+    if picked:
+        models = sorted(m for m in picked if m in counts)
+    else:
+        most = max(counts.values(), default=0)
+        models = sorted(m for m, n in counts.items() if n >= MIN_SHARE * most)
+    if ref in counts and ref not in models:
+        models.append(ref)
+    return [r for r in rows if r["model"] in models], models
+
+
+def default_reference(rows, models, configured=None):
+    """models.toml [defaults] reference when it has answers, else the most accurate model shown."""
+    if configured in models:
+        return configured
+    acc = {m: statistics.fmean(r["ok"] for r in rows if r["model"] == m) for m in models
+           if any(r["model"] == m for r in rows)}
+    return max(acc, key=lambda m: (acc[m], m), default=None)
+
+
+def _pts(x):
+    return f"{100 * x:+.1f}"
+
+
+def diverging(diff, lo, hi, scale, color):
+    """A bar from the centre line to diff, the CI around it as a thin line: -scale .. +scale."""
+    out = []
+    for i in range(2 * HALF + 1):
+        v = (i - HALF) / HALF * scale
+        step = scale / HALF / 2
+        filled = (0 < v <= diff + step) if diff > 0 else (diff - step <= v < 0) if diff < 0 else False
+        if i == HALF:
+            out.append(f"[{color}]█[/]" if filled else "│")
+        elif filled:
+            out.append(f"[{color}]█[/]")
+        elif lo - step <= v <= hi + step:
+            out.append("[#808080]─[/]")
+        else:
+            out.append(" ")
+    return "".join(out)
+
+
+VERDICT_COLOR = {"better": "green", "worse": "red", "tie": "yellow"}
+
+
+def _vs_lines(results, name_of=lambda x: x["model"]):
+    if not results:
+        return []
+    scale = max(0.05, *(max(abs(x["lo"]), abs(x["hi"]), abs(x["diff"])) for x in results))
+    w = max(len(name_of(x)) for x in results)
+    lines = [" " * (w + 2) + f"[dim]{_pts(-scale)} pts" + " " * (2 * HALF - 16) + f"{_pts(scale)} pts[/dim]"]
+    for x in results:
+        color = VERDICT_COLOR[x["verdict"]]
+        lines.append(f"{_esc(name_of(x)):<{w}}  {diverging(x['diff'], x['lo'], x['hi'], scale, color)}  "
+                     f"[{color}]{_pts(x['diff'])} pts[/] [dim]({_pts(x['lo'])} to {_pts(x['hi'])}, "
+                     f"{x['n']} q)[/dim] [{color}]{x['verdict'].upper()}[/]")
+    return lines
+
+
+# ---------------------------------------------------------------- 1 vs reference
+def vs_reference(rows, models, ref):
+    title = "Who's worth keeping: accuracy against the reference"
+    if not ref:
+        return {"title": title, "lines": ["[dim]No reference model with answers.[/dim]"], "actions": []}
+    res = sorted((x for x in compare.vs_reference(rows, ref) if x["pack"] == "all" and x["model"] in models),
+                 key=lambda x: -x["diff"])
+    acc = statistics.fmean(r["ok"] for r in rows if r["model"] == ref) if any(r["model"] == ref for r in rows) else 0
+    lines = [f"[dim]Reference: [b]{_esc(ref)}[/b] ({compare.pct(acc)} right). Paired on the questions both "
+             "answered; the bar is the difference in points, the thin line its 95% CI.[/dim]"] + _vs_lines(res)
+    actions = []
+    for x in res:
+        if x["verdict"] == "better":
+            actions.append(f"{x['model']} beats {ref} by {100 * x['diff']:.1f} pts on {x['n']} shared questions.")
+    worse = [x for x in res if x["verdict"] == "worse"]
+    if worse:
+        actions.append(f"Behind {ref} for certain: " + ", ".join(f"{x['model']} ({_pts(x['diff'])})" for x in worse[:5])
+                       + (f" and {len(worse) - 5} more" if len(worse) > 5 else "") + ": candidates to drop.")
+    same = [x for x in res if x["verdict"] == "tie" and x["lo"] == x["hi"] == 0]
+    if same:
+        actions.append("Same result as " + ref + " on every shared question: " + ", ".join(x["model"] for x in same)
+                       + " (harder questions would separate them).")
+    ties = [x for x in res if x["verdict"] == "tie" and x not in same]
+    if ties:
+        actions.append("Too close to call against " + ref + ": " + ", ".join(
+            f"{x['model']} ({_pts(x['diff'])}, CI {_pts(x['lo'])} to {_pts(x['hi'])})" for x in ties[:4])
+            + ": more questions or a Certify run would decide.")
+    return {"title": title, "lines": lines, "actions": actions}
+
+
+# ---------------------------------------------------------------- 2 use cases
+HEAT = ((0.9, "bold green"), (0.75, "bold yellow"), (0.5, "bold #ff8c00"), (0.0, "bold red"))
+CELL = 12
+
+
+def _heat(acc):
+    return next(style for floor, style in HEAT if acc >= floor)
+
+
+def use_cases(rows, models, packs):
+    """packs: {name: Pack} (for groups and question counts)."""
+    title = "Best model per use case"
+    groups = []
+    for name, p in packs.items():
+        if p.group not in groups:
+            groups.append(p.group)
+    groups = [g for g in groups if any(packs.get(r["suite"]) and packs[r["suite"]].group == g for r in rows)]
+    if not groups:
+        return {"title": title, "lines": ["[dim]No results yet.[/dim]"], "actions": []}
+    size = {g: sum(len(p.tests) for p in packs.values() if p.group == g) for g in groups}
+    per = collections.defaultdict(lambda: collections.defaultdict(list))     # (model, group) -> test -> oks
+    for r in rows:
+        if r["suite"] in packs:
+            per[r["model"], packs[r["suite"]].group][r["test"]].append(r["ok"])
+    cell = {}
+    for m in models:
+        for g in groups:
+            tests = per.get((m, g))
+            if tests:
+                cell[m, g] = (statistics.fmean(statistics.fmean(v) for v in tests.values()), len(tests) / max(1, size[g]))
+    best = {}
+    for g in groups:
+        full = {m: a for (m, gg), (a, cov) in cell.items() if gg == g and cov >= COVERED}
+        if full:
+            top = max(full.values())
+            best[g] = (sorted((m for m, a in full.items() if a == top)), sorted(full.items(), key=lambda x: -x[1]))
+    w = max(len(m) for m in models)
+    short = {g: (g if len(g) <= CELL - 1 else f"{groups.index(g) + 1} {g[:CELL - 4]}…") for g in groups}
+    lines = [" " * (w + 2) + "".join(f"[b]{short[g]:<{CELL}}[/b]" for g in groups)]
+    for m in models:
+        cells = []
+        for g in groups:
+            if (m, g) not in cell:
+                cells.append(f"[#808080]{'-':^{CELL - 1}}[/] ")
+                continue
+            a, cov = cell[m, g]
+            star = "★" if g in best and m in best[g][0] else " "
+            text = f"██ {100 * a:>3.0f}%{'◔' if cov < COVERED else ' '}{star}"
+            cells.append(f"[{_heat(a)}]{text:<{CELL - 1}}[/] ")
+        lines.append(f"{_esc(m):<{w}}  " + "".join(cells))
+    lines.append("[dim]★ best (among models that answered most of it) · ◔ answered under "
+                 f"{100 * COVERED:.0f}% of its questions ·[/dim] " + " ".join(f"[{st}]██[/] [dim]{lbl}[/dim]" for (_, st), lbl in
+                                                                zip(HEAT, ("90%+", "75%+", "50%+", "under 50%"))))
+    if any(short[g] != g for g in groups):
+        lines.append("[dim]" + " · ".join(f"{i + 1} {g}" for i, g in enumerate(groups) if short[g] != g) + "[/dim]")
+    actions = []
+    for g in groups:
+        if g not in best:
+            actions.append(f"{g}: no model has answered most of its questions yet.")
+            continue
+        tops, ranked = best[g]
+        top = ranked[0][1]
+        names = " or ".join(tops[:2]) + (f" (and {len(tops) - 2} more tied)" if len(tops) > 2 else "")
+        line = f"{g}: use {names} ({compare.pct(top)})"
+        rest = [x for x in ranked if x[0] not in tops]
+        if rest:
+            gap = top - rest[0][1]
+            line += f"; next {rest[0][0]} ({compare.pct(rest[0][1])}, {100 * gap:.0f} pts behind)"
+            if gap < 0.05:
+                line += ": not a clear winner"
+        if top < 0.5:
+            line += ". No model gets half of it right"
+        actions.append(line + ".")
+    return {"title": title, "lines": lines, "actions": actions}
+
+
+# ---------------------------------------------------------------- 3 vs base
+def vs_base(rows, models, bases, packs):
+    """bases: {model: its base model's label} (models.toml `base`)."""
+    title = "Fine-tunes against their base model"
+    pairs = [(m, bases[m]) for m in models if bases.get(m) and any(r["model"] == bases[m] for r in rows)]
+    if not pairs:
+        missing = [m for m in models if bases.get(m)]
+        why = (f"The base models of {', '.join(missing)} have no answers to compare with."
+               if missing else 'Add base = "<label of its base model>" to a fine-tune in models.toml to see what it '
+                               "gained or lost against its base.")
+        return {"title": title, "lines": [f"[dim]{_esc(why)}[/dim]"], "actions": []}
+    lines, actions = [], []
+    label = lambda p: packs[p].label if p in packs else p
+    for m, base in pairs:
+        two = [r for r in rows if r["model"] in (m, base)]
+        res = [x for x in compare.vs_reference(two, base) if x["model"] == m]
+        overall = next((x for x in res if x["pack"] == "all"), None)
+        if not overall:
+            continue
+        color = VERDICT_COLOR[overall["verdict"]]
+        lines.append(f"[b]{_esc(m)}[/b] vs [b]{_esc(base)}[/b]: [{color}]{_pts(overall['diff'])} pts "
+                     f"{overall['verdict'].upper()}[/] [dim]({overall['n']} shared questions)[/dim]")
+        lines += ["  " + line for line in _vs_lines([x for x in res if x["pack"] != "all"], lambda x: label(x["pack"]))]
+        cats = categories(two, m, base)
+        lost = [c for c in cats if c[1] <= -0.34][:3]
+        won = [c for c in sorted(cats, key=lambda c: -c[1]) if c[1] >= 0.34][:3]
+        if lost:
+            lines.append("  [red]lost on:[/] " + " · ".join(f"{_esc(c)} {_pts(d)} pts ({n} q)" for c, d, n in lost))
+        if won:
+            lines.append("  [green]gained on:[/] " + " · ".join(f"{_esc(c)} {_pts(d)} pts ({n} q)" for c, d, n in won))
+        lines.append("")
+        worse = [x for x in res if x["pack"] != "all" and x["verdict"] == "worse"]
+        better = [x for x in res if x["pack"] != "all" and x["verdict"] == "better"]
+        if worse:
+            actions.append(f"{m} is worse than its base on " + ", ".join(f"{label(x['pack'])} ({_pts(x['diff'])})"
+                                                                         for x in worse)
+                           + (f"; it lost most on {', '.join(c for c, _, _ in lost)}: gaps in its fine-tuning data"
+                              if lost else "") + ".")
+        if better:
+            actions.append(f"{m} beats its base on " + ", ".join(f"{label(x['pack'])} ({_pts(x['diff'])})" for x in better)
+                           + ".")
+        if not worse and not better:
+            actions.append(f"{m}: no clear difference from {base} on any pack yet"
+                           + (f" (losses on {', '.join(c for c, _, _ in lost)} are worth a look)" if lost else "") + ".")
+    return {"title": title, "lines": lines[:-1] if lines and not lines[-1] else lines, "actions": actions}
+
+
+def categories(rows, m, base, min_q=2):
+    """[(category, mean pass-rate difference m - base, questions)] over categories with at least min_q
+    questions both answered, worst first. A question's category is its `category`, else its pack."""
+    by = collections.defaultdict(lambda: collections.defaultdict(list))
+    cat = {}
+    for r in rows:
+        by[r["model"]][r["test"]].append(r["ok"])
+        cat[r["test"]] = (r.get("raw") or {}).get("category") or r["suite"]
+    per = collections.defaultdict(list)
+    for t in set(by[m]) & set(by[base]):
+        per[cat[t]].append(statistics.fmean(by[m][t]) - statistics.fmean(by[base][t]))
+    return sorted(((c, statistics.fmean(v), len(v)) for c, v in per.items() if len(v) >= min_q), key=lambda c: c[1])
+
+
+# ---------------------------------------------------------------- 4 why they fail
+KINDS = (("right", "green", "█"), ("wrong", "red", "█"), ("cut off", "yellow", "█"),
+         ("no answer / format", "magenta", "█"), ("code crashed or hung", "cyan", "█"), ("critical", "#ff5faf", "▓"))
+FORMAT = ("No 'ANSWER:' line", "Empty answer", "No code", "no tool call")
+CRASHED = ("Timed out after", "Harness crashed", "grader error")
+
+
+def kind(r):
+    if r["ok"]:
+        return "right"
+    if r.get("severity") == "critical":
+        return "critical"
+    if r["truncated"] or r["reason"].startswith("TRUNCATED"):
+        return "cut off"
+    if r["reason"].startswith(FORMAT):
+        return "no answer / format"
+    if r["reason"].startswith(CRASHED):
+        return "code crashed or hung"
+    return "wrong"
+
+
+def _split(counts, total, width):
+    """Blocks per kind, largest remainders first, so they add up to width."""
+    raw = {k: width * counts.get(k, 0) / total for k, *_ in KINDS}
+    out = {k: int(v) for k, v in raw.items()}
+    for k in sorted(raw, key=lambda k: -(raw[k] - out[k]))[:width - sum(out.values())]:
+        out[k] += 1
+    for k in out:       # a kind that happened shows at least one block
+        if counts.get(k) and not out[k]:
+            big = max(out, key=out.get)
+            out[big], out[k] = out[big] - 1, 1
+    return out
+
+
+def failure_mix(rows, models, width=36):
+    title = "Why models fail"
+    counts = {m: collections.Counter(kind(r) for r in rows if r["model"] == m) for m in models}
+    counts = {m: c for m, c in counts.items() if c}
+    if not counts:
+        return {"title": title, "lines": ["[dim]No answers yet.[/dim]"], "actions": []}
+    w = max(len(m) for m in counts)
+    lines = [" " * (w + 2) + "  ".join(f"[{c}]{ch}[/] {k}" for k, c, ch in KINDS)]
+    actions = []
+    for m in sorted(counts, key=lambda m: -counts[m]["right"] / sum(counts[m].values())):
+        c = counts[m]
+        total = sum(c.values())
+        blocks = _split(c, total, width)
+        bar = "".join(f"[{color}]{ch * blocks[k]}[/]" for k, color, ch in KINDS)
+        fails = total - c["right"]
+        detail = ", ".join(f"{c[k]} {k}" for k, *_ in KINDS[1:] if c[k])
+        lines.append(f"{_esc(m):<{w}}  {bar}  {compare.pct(c['right'] / total)} right [dim]({total} answers"
+                     + (f"; {detail}" if detail else "") + ")[/dim]")
+        if fails >= 3:
+            top, n = max(((k, c[k]) for k, *_ in KINDS[1:-1]), key=lambda x: x[1])
+            if n / fails >= 0.4 and top != "wrong":
+                actions.append(f"{m}: {n} of its {fails} failures are {top}: " + {
+                    "cut off": "it thinks too long for max_tokens, rather than reasoning wrong.",
+                    "no answer / format": "it doesn't follow the answer format (check its chat template and the "
+                                          "pack's instructions).",
+                    "code crashed or hung": "its code crashes or never finishes."}[top])
+    crit = sorted(((m, c["critical"]) for m, c in counts.items() if c["critical"]), key=lambda x: -x[1])
+    if crit:
+        actions.insert(0, "Critical failures: " + ", ".join(f"{m} ({n})" for m, n in crit[:5])
+                       + (f" and {len(crit) - 5} more" if len(crit) > 5 else "")
+                       + ": not safe where those matter, whatever the accuracy (Failures lists them).")
+    return {"title": title, "lines": lines, "actions": actions}
+
+
+# ---------------------------------------------------------------- 5 accuracy / cost
+MARKS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+
+def cost_points(rows, models, by="tokens"):
+    """({model: (accuracy, median cost per question)}, questions, shared): over the questions every model
+    answered when there are at least 5 of them, else each model over its own."""
+    tests = compare.per_question(rows)
+    shared = [t for t in tests if all(m in t["cells"] for m in models)]
+    use = shared if len(shared) >= 5 else tests
+    out = {}
+    for m in models:
+        cells = [t["cells"][m] for t in use if m in t["cells"]]
+        answers = sum(c["n"] for c in cells)
+        cost = compare.median(c[by] for c in cells)
+        if answers and cost:
+            out[m] = (sum(c["passed"] for c in cells) / answers, cost)
+    return out, len(use), use is shared
+
+
+def front(points):
+    """The models no other model beats on both accuracy and cost."""
+    return {m for m, (a, c) in points.items()
+            if not any(a2 >= a and c2 <= c and (a2 > a or c2 < c) for m2, (a2, c2) in points.items() if m2 != m)}
+
+
+def accuracy_cost(rows, models, by="tokens", width=56, height=12, hosted=()):
+    """hosted: models served by an API elsewhere; with local ones too, the local trade-offs get a line."""
+    unit = "tokens" if by == "tokens" else "seconds"
+    title = f"Accuracy against {unit} per question"
+    pts, n, shared = cost_points(rows, models, "tokens" if by == "tokens" else "secs")
+    if len(pts) < 2:
+        return {"title": title, "lines": ["[dim]Needs two models with answers.[/dim]"], "actions": []}
+    best = front(pts)
+    order = sorted(pts, key=lambda m: (-pts[m][0], pts[m][1]))
+    mark = {m: MARKS[i % len(MARKS)] for i, m in enumerate(order)}
+    accs, costs = [a for a, _ in pts.values()], [c for _, c in pts.values()]
+    a_lo = math.floor(min(accs) * 10) / 10
+    a_lo = min(a_lo, 0.9)
+    c_lo, c_hi = math.log(min(costs)), math.log(max(costs))
+    grid = [[" "] * width for _ in range(height)]
+    for m, (a, c) in pts.items():
+        x = 0 if c_hi == c_lo else round((math.log(c) - c_lo) / (c_hi - c_lo) * (width - 1))
+        y = round((1 - (a - a_lo) / (1 - a_lo)) * (height - 1)) if a_lo < 1 else 0
+        free = next((xx for xx in (x, x + 1, x - 1) if 0 <= xx < width and grid[y][xx] == " "), None)
+        if free is None:
+            grid[y][x] = "[b]*[/b]"
+        else:
+            grid[y][free] = f"[bold green]{mark[m]}[/]" if m in best else mark[m]
+    fmt = (lambda v: f"{v:,.0f}") if by == "tokens" else (lambda v: f"{v:.0f}s" if v >= 10 else f"{v:.1f}s")
+    lines = []
+    for i, row in enumerate(grid):
+        label = "100%" if i == 0 else f"{100 * a_lo:.0f}%" if i == height - 1 else ""
+        lines.append(f"[dim]{label:>5} │[/dim]" + "".join(row))
+    lo_s, hi_s = fmt(min(costs)), fmt(max(costs))
+    lines.append("[dim]      └" + "─" * width + "[/dim]")
+    lines.append(f"[dim]       {lo_s}{' ' * max(1, width - len(lo_s) - len(hi_s))}{hi_s}  median {unit} "
+                 "per question (log scale) →[/dim]")
+    lines.append(f"[dim]Over {n} questions {'every model answered' if shared else 'each model answered (too few in common)'}. "
+                 "[bold green]Green[/] = no other model is both more accurate and cheaper. * = several models.[/dim]")
+    for m in order:
+        a, c = pts[m]
+        tag = f"[bold green]{mark[m]}[/]" if m in best else mark[m]
+        lines.append(f"  {tag} {_esc(m)}  [dim]{compare.pct(a)} · {fmt(c)}{' · hosted' if m in hosted else ''}[/dim]")
+    beaten = collections.defaultdict(list)     # front model -> the models it beats on both
+    for m in order:
+        if m in best:
+            continue
+        a, c = pts[m]
+        better = [(m2, c2) for m2, (a2, c2) in pts.items() if m2 in best and a2 >= a and c2 <= c]
+        if better:
+            m2, c2 = min(better, key=lambda x: x[1])
+            beaten[m2].append(f"{m} ({100 * c2 / c:.0f}% of its {unit}"
+                              + (", same accuracy)" if pts[m2][0] == a else ")"))
+    actions = ["Best trade-offs: " + ", ".join(m for m in order if m in best) + "."]
+    local = {m: p for m, p in pts.items() if m not in hosted}
+    if hosted and len(local) >= 2 and len(local) < len(pts):
+        lf = front(local)
+        actions.append("Among local models: " + ", ".join(m for m in order if m in lf) + ".")
+    for m2, items in beaten.items():
+        actions.append(f"{m2} is at least as accurate with fewer {unit} than " + ", ".join(items[:4])
+                       + (f" and {len(items) - 4} more" if len(items) > 4 else "") + ".")
+    return {"title": title, "lines": lines, "actions": actions}
+
+
+# ---------------------------------------------------------------- 6 question health
+def question_health(rows, models, min_models=3, width=40):
+    title = "Question health"
+    need = min(min_models, len(models))
+    tests = [t for t in compare.per_question(rows) if len(t["cells"]) >= max(2, need)]
+    if not tests:
+        return {"title": title, "lines": [f"[dim]Needs questions answered by at least {max(2, need)} models.[/dim]"],
+                "actions": []}
+    rate = lambda t: statistics.fmean(c["passed"] / c["n"] for c in t["cells"].values())
+    none = [t for t in tests if all(c["passed"] == 0 for c in t["cells"].values())]
+    every = [t for t in tests if all(c["passed"] == c["n"] for c in t["cells"].values())]
+    split = sorted((t for t in tests if t not in none and t not in every),
+                   key=lambda t: (abs(rate(t) - 0.5), -len(t["cells"])))
+    n = len(tests)
+    blocks = _split({"wrong": len(none), "right": len(every), "cut off": len(split)}, n, width)
+    bar = f"[red]{'█' * blocks['wrong']}[/][yellow]{'█' * blocks['cut off']}[/][#808080]{'█' * blocks['right']}[/]"
+    lines = [f"{bar}  [dim]{n} questions answered by {max(2, need)}+ models[/dim]",
+             f"[yellow]█[/] {len(split)} split models  [red]█[/] {len(none)} every model fails  "
+             f"[#808080]█[/] {len(every)} every model passes"]
+    desc = lambda t: (t["cells"][next(iter(t["cells"]))]["rows"][0].get("raw") or {}).get("description") or t["name"]
+    for name, items, color in (("Split models most", split, "yellow"), ("Every model fails", none, "red"),
+                               ("Every model passes", every, "#808080")):
+        if items:
+            lines.append(f"[{color}]{name}:[/]")
+            for t in items[:5]:
+                lines.append(f"  [dim]{_esc(t['suite'])}[/dim] {_esc(desc(t)[:70])}"
+                             + (f" [dim]({100 * rate(t):.0f}% of answers right)[/dim]" if items is split else ""))
+            if len(items) > 5:
+                lines.append(f"  [dim]… and {len(items) - 5} more[/dim]")
+    actions = []
+    if none:
+        actions.append(f"{len(none)} question{'s' * (len(none) > 1)} no model passes: check their expected answers "
+                       "(or keep them as stretch goals).")
+    if every:
+        secs = statistics.median(c["secs"] for t in every for c in t["cells"].values() if c["secs"]) \
+            if any(c["secs"] for t in every for c in t["cells"].values()) else None
+        saved = f", saving ~{_dur(secs * len(every))} per model" if secs else ""
+        actions.append(f"{len(every)} question{'s' * (len(every) > 1)} every model passes: they don't separate models; "
+                       f"move them to a regression pack{saved}.")
+    if split:
+        actions.append(f"{len(split)} question{'s' * (len(split) > 1)} split models: these decide rankings; "
+                       "write more like the top ones.")
+    return {"title": title, "lines": lines, "actions": actions}
+
+
+def _dur(secs):
+    secs = int(secs)
+    return f"{secs // 3600}h{secs % 3600 // 60:02d}m" if secs >= 3600 else f"{secs // 60}m" if secs >= 60 else f"{secs}s"
+
+
+# ---------------------------------------------------------------- 7 next runs
+def next_runs(engine, models, ranking, limit=4, skip=()):
+    """ranking: models best first (by the reference comparison, else accuracy). The Certify runs that
+    would settle the open verdicts of the strongest `limit` models, with their time."""
+    from . import verdict
+    title = "Next runs"
+    table = verdict.readiness(engine, models)
+    times = engine.answer_times()
+    med = {l: {p: statistics.median(ts) for p, ts in pp.items() if ts} for l, pp in times.items()}
+    lines, actions, total = [], [], 0
+    shown = 0
+    for m in ranking:
+        if m not in table or m in skip or shown >= limit:
+            continue
+        todo = []
+        for g, (v, packs) in table[m].items():
+            for name, pv in packs.items():
+                ev = pv.evidence
+                if pv.status != "INCONCLUSIVE" or not ev or ev.get("certified"):
+                    continue
+                left = max(0, ev["tests_total"] * ev["want_repeat"] - ev["trials"])
+                spa = med.get(m, {}).get(name) or statistics.median(
+                    [x[name] for x in med.values() if name in x] or [60.0])
+                todo.append((name, left, left * spa))
+        if not todo:
+            continue
+        shown += 1
+        answers, secs = sum(x[1] for x in todo), sum(x[2] for x in todo)
+        label = ", ".join(engine.packs[n].label if n in engine.packs else n for n, *_ in todo)
+        lines.append(f"[b]{shown}. Certify {_esc(m)}[/b] on {_esc(label)}: ~{answers} answers, ~{_dur(secs)}")
+        lines.append(f"   [dim]tuieval run --tier certify --only {_esc(m)} --packs {','.join(n for n, *_ in todo)}[/dim]")
+        total += secs
+    if shown:
+        actions.append(f"Certifying {'these ' + str(shown) + ' contenders' if shown > 1 else 'it'} takes ~{_dur(total)} "
+                       "in all; the rest are behind them.")
+    if not lines:
+        lines = ["[dim]Nothing open among the strongest models: their verdicts are settled or not run yet.[/dim]"]
+    return {"title": title, "lines": lines, "actions": actions}
+
+
+# ---------------------------------------------------------------- all of them
+def ranking(rows, models, ref):
+    """Models best first: by the difference to the reference (the reference first), else by accuracy."""
+    diff = {x["model"]: x["diff"] for x in compare.vs_reference(rows, ref) if x["pack"] == "all"} if ref else {}
+    acc = {m: statistics.fmean(r["ok"] for r in rows if r["model"] == m) for m in models
+           if any(r["model"] == m for r in rows)}
+    key = (lambda m: -(diff.get(m, 0) if m != ref else 0)) if diff else (lambda m: -acc.get(m, 0))
+    return sorted(acc, key=lambda m: (key(m), m))
+
+
+def panels(rows, models, packs, ref=None, bases=None, by="tokens", engine=None):
+    """All seven (the last needs the engine: verdicts and answer times). Models are shown best first."""
+    order = ranking(rows, models, ref)
+    order += [m for m in models if m not in order]
+    hosted = {m["label"] for m in engine.cfg["models"] if engine.hosted(m)} if engine is not None else set()
+    out = [vs_reference(rows, order, ref), use_cases(rows, order, packs), vs_base(rows, order, bases or {}, packs),
+           failure_mix(rows, order), accuracy_cost(rows, order, by, hosted=hosted), question_health(rows, order)]
+    if engine is not None:
+        out.append(next_runs(engine, order, order, skip={ref}))
+    return out
+
+
+def render(panel, number=None):
+    """A panel as Rich markup: title, chart, then its action lines."""
+    head = f"[b reverse] {f'{number} ' if number else ''}{panel['title']} [/]"
+    acts = [f"[bold cyan]→[/] {_esc(a)}" for a in panel["actions"][:MAX_ACTIONS]]
+    return "\n".join([head, ""] + panel["lines"] + ([""] + acts if acts else []))

tuieval/pta.py

@@ -47,7 +47,7 @@ def memory_gb(infos):
 
 
 def index(rows, models=None, memory=None):
-    """{"questions": n shared, "models": [{model, P, T, A, tokens, total_s, passed, answers, alongside,
+    """{"questions": n shared, "models": [{model, P, T, A, cut, cut_pct, tokens, total_s, passed, answers, alongside,
     fewest, fastest, tps, ttft, memory_gb}],
     "left_out": [(model, questions answered)], "no_answers": [model], "most": questions answered by the
     most-answered model}, best accuracy first. models: the ones to compare (default: every model in rows
@@ -75,6 +75,9 @@ def index(rows, models=None, memory=None):
                     "total_s": sum(c["secs"] or 0 for c in cells),
                     "passed": sum(c["passed"] for c in cells), "answers": answers,
                     "A": 100 * sum(c["passed"] for c in cells) / answers if answers else None,
+                    # failed only because max_tokens ran out (overthinking, not wrong yet)
+                    "cut": sum(c["truncated"] for c in cells),
+                    "cut_pct": 100 * sum(c["truncated"] for c in cells) / answers if answers else None,
                     # generation speed and time to first token over the same answers (medians)
                     "tps": compare.median(r.get("gen_tps") for c in cells for r in c["rows"]),
                     "ttft": compare.median(r.get("ttft") for c in cells for r in c["rows"]),
@@ -118,7 +121,8 @@ def bars(result, width=20, markup=True):
     """Lines with a row per model and a bar each for P, T and A: the at-a-glance view. The P and T bars
     are proportional to tokens and time (the longest is the model that used most; shorter is better),
     with how many times the leanest / fastest model's beside them; the A bar is the share of answers
-    right (longer is better, full only when every answer passed). markup=False gives plain text."""
+    right (longer is better, full only when every answer passed), then the share cut off at max_tokens in
+    orange (▓ in plain text) and "✂ n%" beside it. markup=False gives plain text."""
     if not result["models"]:
         return []
     name_w = max(len(x["model"]) for x in result["models"])
@@ -138,12 +142,18 @@ def bars(result, width=20, markup=True):
                 if v is not None and v < 100:   # full only when every answer passed
                     full = min(full, width - 1)
                 num = "-" if v is None else compare.pct(v / 100)
+                cut = x.get("cut_pct") or 0
+                part = min(width - full, max(1, round(width * cut / 100))) if cut else 0
+                if cut:   # after the accuracy, which stays aligned with the other rows'
+                    num = f"{num:>6}  ✂ {compare.pct(cut / 100)}"
             else:
                 full = _blocks(width, x.get(measure), most[measure])
                 num = times(x.get(shown))
-            bar, rest = "█" * full, "░" * (width - full)
-            bar = f"[{color}]{bar}[/][grey37]{rest}[/]" if markup else bar + rest
-            cells.append(f"{bar} {num:>6}" + " " * (cell_w - width - 7))
+                part = 0
+            bar, cut_bar, rest = "█" * full, "▓" * part, "░" * (width - full - part)
+            bar = (f"[{color}]{bar}[/]" + (f"[#ff8c00]{'█' * part}[/]" if part else "") + f"[grey37]{rest}[/]"
+                   if markup else bar + cut_bar + rest)
+            cells.append(f"{bar} {num:>6}" + " " * max(0, cell_w - width - 1 - max(6, len(num))))
         name = f"{x['model']:<{name_w}}"
         lines.append((f"{rich_escape(name) if markup else name}  " + "".join(cells)).rstrip())
     return lines
@@ -168,10 +178,11 @@ def left_out_lines(result):
 def table(result):
     """(header, rows) for a plain table of the index."""
     n = result["questions"] or 1
-    header = ["model", "tokens/answer", "time/answer", "answers right", "tok/s", "TTFT s", "memory GB"]
+    header = ["model", "tokens/answer", "time/answer", "answers right", "cut off", "tok/s", "TTFT s", "memory GB"]
     rows = [[x["model"], f"{round(x['tokens'] / n):,}" + (" ★" if x.get("fewest") else ""),
              fmt_secs(x["total_s"] / n) + (" ★" if x.get("fastest") else ""),
-             f"{x['passed']}/{x['answers']}", _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
+             f"{x['passed']}/{x['answers']}", f"{x.get('cut') or 0}/{x['answers']}",
+             _num(x.get("tps"), ".0f"), _num(x.get("ttft"), ".1f"),
              _num(x.get("memory_gb"), ".1f")] for x in result["models"]]
     return header, rows
 
@@ -204,7 +215,9 @@ def scope_note(result, models):
     return (f"Compared on the {result['questions']} questions all {len(result['models'])} model(s) answered. "
             "P parsimony (tokens) and T time: bars as long as the tokens / time used (shorter is better), "
             "and how many times the leanest / fastest model's (1.0×, ★ in the table). A accuracy: answers right "
-            "(longer is better).")
+            "(longer is better)"
+            + ("; orange ✂: answers cut off at max_tokens, out of tokens rather than wrong (they count as wrong)."
+               if any(x.get("cut") for x in result["models"]) else "."))
 
 
 def for_engine(e, labels=None, packs=None, results_dir=None):

tuieval/run_evals.py

@@ -427,6 +427,42 @@ def cmd_pta(argv):
         out.print(line, markup=False)
 
 
+def cmd_insights(argv):
+    p = argparse.ArgumentParser(prog="tuieval insights",
+                                description="Charts with what to do next: every model against a reference, the best "
+                                            "model per use case, fine-tunes against their base, why models fail, "
+                                            "accuracy against cost, question health and the runs that would settle "
+                                            "open verdicts.")
+    p.add_argument("--only", help="comma-separated model labels (default: every model with enough answers)")
+    p.add_argument("--reference", help="the model to compare against (default: [defaults] reference, else the "
+                                       "most accurate)")
+    p.add_argument("--packs", help="comma-separated packs (default: all)")
+    p.add_argument("--cost", choices=("tokens", "time"), default="tokens", help="the cost axis of the accuracy chart")
+    p.add_argument("--models")
+    p.add_argument("--results-dir")
+    a = p.parse_args(argv)
+    from rich.console import Console
+    from . import compare, insights
+    e = engine.Engine(models_path=a.models, results_dir=a.results_dir)
+    paths = [x for x in compare.default_paths(e.results_dir) if not e.superseded(x)]
+    rows, _ = compare.load(paths) if paths else ([], [])
+    if a.packs:
+        rows = [r for r in rows if r["suite"] in {x.strip() for x in a.packs.split(",")}]
+    wanted = a.reference or e.cfg["defaults"].get("reference")
+    picked = {x.strip() for x in a.only.split(",")} if a.only else None
+    rows, models = insights.prepare(rows, picked, wanted)
+    if not models:
+        sys.exit("no finished results" + (" for those models or packs" if a.only or a.packs else ""))
+    if a.reference and a.reference not in models:
+        sys.exit(f"{a.reference} has no finished results")
+    ref = insights.default_reference(rows, models, wanted)
+    bases = {m["label"]: m["base"] for m in e.cfg["models"] if m.get("base")}
+    out = Console(highlight=False)
+    for i, panel in enumerate(insights.panels(rows, models, e.packs, ref, bases, a.cost, engine=e), 1):
+        out.print(insights.render(panel, i), soft_wrap=True)
+        out.print()
+
+
 def cmd_report(argv):
     p = argparse.ArgumentParser(prog="tuieval report", description="Write the readiness report as markdown.")
     p.add_argument("-o", "--output", help="default: reports/readiness-<date>.md")
@@ -708,7 +744,7 @@ def cmd_export(argv):
 COMMANDS = {"add": cmd_add, "list": cmd_list, "scan": cmd_scan, "regrade": cmd_regrade,
             "verdict": cmd_verdict, "report": cmd_report, "items": cmd_items, "selftest": cmd_selftest,
             "capture": cmd_capture, "history": cmd_history, "machines": cmd_machines, "tune": cmd_tune,
-            "export": cmd_export, "remove": cmd_remove, "pta": cmd_pta}
+            "export": cmd_export, "remove": cmd_remove, "pta": cmd_pta, "insights": cmd_insights}
 
 
 def parse_tests(text, pack_names, known):

tuieval/tui.py

@@ -2690,6 +2690,14 @@ class ResultsScreen(Screen):
                 yield Static(id="pta-bars")
                 yield DataTable(id="pta", zebra_stripes=True, cursor_type="none")
                 yield Static(id="pta-left-out")
+            with TabPane("Insights", id="tab-insights"):
+                with Horizontal(id="ins-controls"):
+                    yield Select([], id="ins-ref", prompt="Reference model", allow_blank=True)
+                    yield Select([("Cost: tokens", "tokens"), ("Cost: time", "time")], id="ins-cost",
+                                 allow_blank=False, value="tokens")
+                    yield Button("Models: all", id="ins-models")
+                with VerticalScroll(id="ins-body"):
+                    yield Static(id="ins-panels")
             with TabPane("Production readiness"):
                 yield Static(id="readiness-short")
                 yield Static(self.READINESS_HELP, id="readiness-help")
@@ -2799,6 +2807,9 @@ class ResultsScreen(Screen):
         self.fill_per_question()
         self.fill_pta()
         self.fill_readiness(results_dir)
+        self.insights_stale = True
+        if self.query_one(TabbedContent).active == "tab-insights":
+            self.fill_insights()
         self.sort_tables()
 
     @on(DataTable.RowSelected, "#failures")
@@ -2863,6 +2874,7 @@ class ResultsScreen(Screen):
 
     @on(Button.Pressed, "#pq-models")
     @on(Button.Pressed, "#pta-models")
+    @on(Button.Pressed, "#ins-models")
     def pick_models(self):
         counts = {}
         for t in getattr(self, "pq_all_tests", []):
@@ -2881,8 +2893,55 @@ class ResultsScreen(Screen):
             # after the picker has closed: a table rebuilt under a modal keeps its header-only widths
             self.call_after_refresh(self.fill_per_question)
             self.call_after_refresh(self.fill_pta)
+            self.insights_stale = True
+            if self.query_one(TabbedContent).active == "tab-insights":
+                self.call_after_refresh(self.fill_insights)
         self.app.push_screen(ModelPickScreen(models, counts, self.app.pq_models), picked)
 
+    # -- insights: charts with what to do next (insights.py); built when the tab is first shown
+    @on(TabbedContent.TabActivated)
+    def tab_activated(self, event):
+        if event.pane.id == "tab-insights" and getattr(self, "insights_stale", True):
+            self.fill_insights()
+
+    @on(Select.Changed, "#ins-ref")
+    @on(Select.Changed, "#ins-cost")
+    def insights_changed(self, event):
+        if event.select.id == "ins-ref" and event.value != Select.BLANK:
+            self.app.insights_ref = event.value   # kept for this session
+        if getattr(self, "rows", None) is not None and not getattr(self, "filling_insights", False):
+            self.fill_insights()
+
+    def fill_insights(self):
+        from . import insights
+        e = self.app.engine
+        self.insights_stale = False
+        rows = [r for r in self.rows if r.get("path") not in self.superseded]
+        body = self.query_one("#ins-panels", Static)
+        if not rows:
+            body.update("No finished results yet.")
+            return
+        sel = self.query_one("#ins-ref", Select)
+        every = sorted({r["model"] for r in rows if not r.get("error")})
+        wanted = getattr(self.app, "insights_ref", None) or e.cfg["defaults"].get("reference")
+        rows, models = insights.prepare(rows, self.app.pq_models, wanted if wanted in every else None)
+        ref = insights.default_reference(rows, models, wanted)
+        self.filling_insights = True
+        try:
+            with sel.prevent(Select.Changed):
+                sel.set_options([(m, m) for m in every])
+                if ref:
+                    sel.value = ref
+        finally:
+            self.filling_insights = False
+        chosen = {m for m in self.app.pq_models if m in every}
+        self.query_one("#ins-models", Button).label = f"Models: {len(chosen)} picked" if chosen else "Models: all"
+        main = os.path.abspath(self.results_dir or e.results_dir) == os.path.abspath(e.results_dir)
+        bases = {m["label"]: m["base"] for m in e.cfg["models"] if m.get("base")}
+        by = self.query_one("#ins-cost", Select).value or "tokens"
+        out = insights.panels(rows, models, e.packs, ref, bases, by, engine=e if main else None)
+        body.update("\n\n".join(insights.render(p, i) for i, p in enumerate(out, 1)))
+
     @on(Select.Changed, "#pta-pack")
     def pta_changed(self):
         if getattr(self, "rows", None) is not None:
@@ -3206,6 +3265,10 @@ class EvalsApp(App):
     #toggle-smoke { width: auto; min-width: 32; }
     #pq-controls { height: 3; }
     #pta-controls { height: 3; }
+    #ins-controls { height: 3; }
+    #ins-controls Select { width: 40; margin-right: 1; }
+    #ins-controls #ins-cost { width: 20; }
+    #ins-body { height: 1fr; }
     #pta-controls Select { width: 28; margin-right: 1; }
     #pta-bars { height: auto; padding: 1 0 0 0; }
     #pta { width: 1fr; height: auto; margin-top: 1; }

{tuieval-0.2.0.dev92.dist-info → tuieval-0.2.0.dev96.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev92
+Version: 0.2.0.dev96
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev92.dist-info → tuieval-0.2.0.dev96.dist-info}/RECORD

@@ -1,24 +1,25 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=LTyNMvKP3YQzCLuW_fhXY3Ipvz3I7zhbkkf5Q0Fzxqs,535
+tuieval/_version.py,sha256=vq8xfmWFF-z2N4IfHoIiukOF66_VgRgLRvVKdboXuLA,535
 tuieval/builtin.toml,sha256=Nwu7O0qFzcrDAxXO_VZtDPvjVwxZ29OtYOW0BPihZGs,4410
-tuieval/cli.py,sha256=VwiaU1F9bZtU8C-gEowJq58kPeYQY4t6SY_HwrV-qgc,4702
+tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
 tuieval/client.py,sha256=wtACBVsngtWopWWhIc4bB6mEUEi31dvvDePN-PjmXbM,11122
 tuieval/compare.py,sha256=uXU_Yzf4zDSHoupHQrTmBhsfc8wCyI1nMaZQN81ybiA,30522
 tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
 tuieval/engine.py,sha256=96GWzKzMQ_KaG6wXrIuBiKWFCcblXWL9cQs89QYgj7Y,125256
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
+tuieval/insights.py,sha256=m6HZk1VbX-avHr3xyT8eVJ5OLjKa_EKTVBxsu04X_HQ,28947
 tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
 tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
 tuieval/packs.py,sha256=ZgaZ1m2ePYvDjmddSc3nafnMEn7kTq9WcDpCaJSZleQ,12026
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
-tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
+tuieval/pta.py,sha256=_e6coSXYSzEagWJItyhz-DqIud5ywoVVa7BXbS7kUnY,13183
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=FU3bvu3jt9ZWg3YLsXNINbdeClXMioyc4zQ4yEhmBQs,47855
+tuieval/run_evals.py,sha256=DN6FhC8Zj08N5pwlrCmlkbkTxA2vmT-G5sWJnp-S-AI,50114
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=a3ZANtRLq9bBSIdxRj2Ksgj8S5lF8shSMNBxpIzH6rM,178622
+tuieval/tui.py,sha256=1JpAtIpq4zJAmiPyg5NYlQQm8lYuhoP1DiRM4-exHsw,181966
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -47,8 +48,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev92.dist-info/METADATA,sha256=1zUki84oRDfw_gL4VqsvwMsDMxV35smy8dZheMab-RI,14580
-tuieval-0.2.0.dev92.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev92.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev92.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev92.dist-info/RECORD,,
+tuieval-0.2.0.dev96.dist-info/METADATA,sha256=tWhbY2NNalpYs0drNts9Vs1vsPHZ-Muy9Nw-34gftRw,14580
+tuieval-0.2.0.dev96.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev96.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev96.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev96.dist-info/RECORD,,