tuieval 0.2.0.dev72__tar.gz → 0.2.0.dev75__tar.gz

{tuieval-0.2.0.dev72 → tuieval-0.2.0.dev75}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Fix: a question description or grader reason containing `[` no longer crashes the run screen.
+- Retest asks per pack before it starts: from zero, continue or leave out. It now really starts from zero, and replaced smoke results go to history/.
 - Retest: the setup checkbox "Rerun existing results" is now "Retest (ask again, even if already answered)", and `tuieval run --retest` is the same (`--force` still works).
 - `tuieval logs` follows the server log of the model running now and moves on with the run; `logs live` shows each answer as it finishes, `--list` every log.
 - Results tables sort by a click on a column name; a second click reverses. Numbers, times and levels sort by value; empty cells stay last.

{tuieval-0.2.0.dev72 → tuieval-0.2.0.dev75}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev72
+Version: 0.2.0.dev75
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev72 → tuieval-0.2.0.dev75}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev72'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev72')
+__version__ = version = '0.2.0.dev75'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev75')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev72 → tuieval-0.2.0.dev75}/src/tuieval/engine.py

@@ -427,6 +427,7 @@ class Job:
     settings: dict = dataclasses.field(default_factory=dict)  # sampling + serving identity (fingerprinted)
     merge: bool = True      # keep matching earlier results and only run what's missing
     redo: bool = False      # rerun the picked tests, keeping the pack's other answers
+    fresh: bool = False     # retest: an unfinished run's matching answers are set aside too, not resumed
     status: str = "waiting"  # waiting, loading, running, done, failed, skipped
     note: str = ""
     discards: list = dataclasses.field(default_factory=list)  # [(what, answers, why)] set aside on start
@@ -1059,6 +1060,8 @@ class Engine:
                 lines = [json.loads(l) for l in f if l.strip()]
             if lines and self._header_ok(lines[0], job):
                 keys |= {(r["test"], r["repeat"]) for r in lines[1:] if not client.server_error_row(r)}
+        if job.fresh:   # a retest asks these again
+            keys = {k for k in keys if k[0] not in self._redo_ids(job)}
         return keys & wanted
 
     def _stored(self, job):
@@ -1119,6 +1122,20 @@ class Engine:
             out.append(("unfinished", len(rows), why))
         return out
 
+    def set_aside(self, job):
+        """(finished, unfinished): earlier answers a retest of this job asks again, so they move to
+        history/ and stop counting: in its result file and its unfinished run, under the job's
+        questions and settings (only the picked tests' when it redoes those)."""
+        redo = self._redo_ids(job)
+        data = self._result(job.out_path)
+        finished = 0
+        if data and data["pack"]["fingerprint"] == job.pack.fingerprint and settings_match(data, job.settings):
+            finished = sum(r["test"] in redo for r in data["results"])
+        head, rows = self._partial_lines(job.out_path + ".partial.jsonl")
+        unfinished = sum(r["test"] in redo and not client.server_error_row(r) for r in rows) \
+            if self._header_ok(head, job) else 0
+        return finished, unfinished
+
     def superseded(self, path):
         """Is this result file about to be replaced? True when it no longer matches the current
         questions or settings and an unfinished run of the same pack does."""
@@ -1141,7 +1158,8 @@ class Engine:
         screen, the pack's certification repeats for certify). tests: {pack: [test ids]} runs just
         those tests of a pack instead of the tier's sample (unknown ids raise PackError). Earlier
         matching results are kept and only missing answers are run, unless force; force with
-        picked tests reruns those and keeps the pack's other answers."""
+        picked tests reruns those and keeps the pack's other answers. force is True, False, or the
+        keys ("label/pack") of the jobs to retest, the others resuming."""
         if tier not in TIERS:
             raise ValueError(f"tier must be one of {TIERS}")
         jobs = []
@@ -1151,9 +1169,11 @@ class Engine:
                 pack = self.packs[name]
                 reps = repeat or self.default_repeat(pack, tier)
                 picked = (tests or {}).get(name)
+                retest = force if isinstance(force, bool) else f"{label}/{name}" in force
                 job = Job(m, pack, reps, pack.pick(picked) if picked else pack.select(tier),
                           self.result_path(label, name, tier), effective_sampling(self.cfg, m), tier=tier,
-                          merge=not force or bool(picked), redo=force and bool(picked), settings=self.settings(m))
+                          merge=not retest or bool(picked), redo=retest and bool(picked), fresh=retest,
+                          settings=self.settings(m))
                 sv = self.serving(m)
                 status = self.result_status(label, name, tier)
                 job.discards = self._discards(job)
@@ -1171,11 +1191,11 @@ class Engine:
                     job.status, job.note = "skipped", f"{', '.join(missing)} not installed in tuieval's Python"
                 elif status and status.startswith("outdated"):
                     job.merge, job.redo, job.note = False, False, status.split(": ", 1)[-1] + ", rerunning"
-                    if not force:   # a rerun under the new settings already under way resumes
+                    if not retest:   # a rerun under the new settings already under way resumes
                         self._fill_progress(job)
                         if job.done:
                             job.note += f" ({job.done} of {job.total} done)"
-                elif not force:
+                elif not retest:
                     self._fill_progress(job)
                     if job.done >= job.total:
                         job.status, job.note = "done", "done earlier"
@@ -2092,11 +2112,16 @@ class Engine:
                 lines = [json.loads(l) for l in f if l.strip()]
             if lines and self._header_ok(lines[0], job):
                 new = [r for r in lines[1:] if not client.server_error_row(r)]
+                if job.fresh:   # a retest asks again what the unfinished run already answered
+                    stay = [r for r in new if r["test"] not in self._redo_ids(job)]
+                    if len(stay) < len(new):
+                        self._partial_to_history(job, keep=bool(stay))
+                        new = stay
+                        if stay:
+                            with open(partial, "w") as f:
+                                f.write("".join(json.dumps(l) + "\n" for l in [lines[0]] + stay))
             else:   # other questions or settings: kept in history/, not deleted
-                hist = os.path.join(os.path.dirname(job.out_path), "history")
-                os.makedirs(hist, exist_ok=True)
-                stamp = time.strftime("%Y%m%d-%H%M%S", time.localtime(os.path.getmtime(partial)))
-                os.replace(partial, os.path.join(hist, f"{job.pack.name}-{stamp}.partial.jsonl"))
+                self._partial_to_history(job)
         if not os.path.isfile(partial):
             with contextlib.suppress(OSError):   # a fresh start: sittings of a discarded partial don't count
                 os.remove(self._sittings_path(job))
@@ -2224,10 +2249,23 @@ class Engine:
         job.note = f"{job.passed}/{job.done} passed"
         self.emit("job_done", job=job)
 
+    def _partial_to_history(self, job, keep=False):
+        """Keep an unfinished run that is set aside in history/; keep: copy it (some of its answers stay)."""
+        partial = job.out_path + ".partial.jsonl"
+        hist = os.path.join(os.path.dirname(job.out_path), "history")
+        os.makedirs(hist, exist_ok=True)
+        stamp = time.strftime("%Y%m%d-%H%M%S", time.localtime(os.path.getmtime(partial)))
+        (shutil.copy2 if keep else os.replace)(partial, os.path.join(hist, f"{job.pack.name}-{stamp}.partial.jsonl"))
+
+    @staticmethod
+    def _redo_ids(job):
+        """The tests whose earlier answers a retest replaces: the picked ones, or all of them."""
+        return {t["id"] for t in job.tests} if job.redo else {t["id"] for t in job.pack.tests}
+
     def _archive(self, job, keep=False):
         """Keep a result file that is about to be replaced (rerun or outdated) in history/; keep:
-        copy it (some of its answers stay in the new file)."""
-        if job.tier == "smoke" or not os.path.isfile(job.out_path):
+        copy it (some of its answers stay in the new file). Smoke results too: nothing is deleted."""
+        if not os.path.isfile(job.out_path):
             return
         hist = os.path.join(os.path.dirname(job.out_path), "history")
         os.makedirs(hist, exist_ok=True)

{tuieval-0.2.0.dev72 → tuieval-0.2.0.dev75}/src/tuieval/run_evals.py

@@ -823,6 +823,10 @@ def main(argv=None):
             say(f"{j.key}: skipping ({j.note})")
         elif j.status == "done":
             say(f"{j.key}: done earlier ({j.passed}/{j.done} passed)")
+        elif j.fresh and any(e.set_aside(j)):
+            finished, unfinished = e.set_aside(j)
+            what = " and ".join(f"{n} {w}" for n, w in ((finished, "finished"), (unfinished, "unfinished")) if n)
+            say(f"{j.key}: retest from zero; its {what} answers move to history/")
 
     def stop(*_):
         say("stopping…", RED)

{tuieval-0.2.0.dev72 → tuieval-0.2.0.dev75}/src/tuieval/tui.py

@@ -386,7 +386,9 @@ HELP = {
 parallel_models in models.toml, default 1). Answers note the models they ran alongside.
 
 [b]Retest[/b]: ask the ticked questions again even if they're already answered (only the tests
-picked with e, if any). Without it, answered questions are skipped. Replaced answers go to history/.
+picked with e, if any). Without it, answered questions are skipped. Before anything moves, Start lists
+the packs with earlier answers and lets you choose for each: Zero (ask everything again), Continue
+(keep what's answered, ask the rest) or Leave out. Replaced answers go to history/, never deleted.
 
 [b]Keys[/b]
   s  start (with a run going: add to it, or queue it)   r  results
@@ -961,7 +963,9 @@ class SetupScreen(Screen):
         if not run:
             text += "\n[yellow]Nothing selected can run (see skipping).[/yellow]"
         elif rerun:
-            text += "\n[yellow]Everything selected has results; Start retests it.[/yellow]"
+            text += "\n[yellow]Everything selected has results; Start asks which packs to ask again.[/yellow]"
+        if force:
+            text += "\n[yellow]Retest: Start lists the packs that would start from zero and lets you choose.[/yellow]"
         resumed = [j for j in run if "done earlier" in j.note]
         if resumed:
             text += f"   [yellow]{len(resumed)} pack(s) continue from earlier results[/yellow]"
@@ -1040,32 +1044,32 @@ class SetupScreen(Screen):
         picks = self.current_picks(suites)
         sel = {"models": labels, "packs": suites, "repeat": repeat, "tier": tier, "force": force, "tests": picks,
                "parallel": self.parallel()}
-        jobs = self.app.engine.plan(labels, suites, repeat, tier, force, picks)
-        rerun = False
+        e = self.app.engine
+        jobs = e.plan(labels, suites, repeat, tier, True, picks)
         if not any(j.status == "waiting" for j in jobs):
-            # Everything selected already has results: Start means run it again (earlier results
-            # are kept in history/). Packs skipped for another reason (no vision, …) stay skipped.
-            jobs = self.app.engine.plan(labels, suites, repeat, tier, True, picks)
-            if not any(j.status == "waiting" for j in jobs):
-                reasons = sorted({j.note for j in jobs if j.status == "skipped"})
-                self.notify(f"Nothing selected can run: {'; '.join(reasons)}", severity="warning")
-                return
-            rerun = True
-        self.app.save_state(sel)
-        lost = [(j, what, n, why) for j in jobs if j.status == "waiting" for what, n, why in j.discards]
-        item = QueuedRun(sel, rerun, {(j.key, what) for j, what, _, _ in lost}, jobs, self.app.engine)
-        if not lost:
-            self.app.start_or_queue(item)
+            reasons = sorted({j.note for j in jobs if j.status == "skipped"})
+            self.notify(f"Nothing selected can run: {'; '.join(reasons)}", severity="warning")
             return
+        self.app.save_state(sel)
+        # Packs whose earlier answers a start would set aside (Retest, everything already done, or
+        # other questions or settings): the user picks per pack before anything moves.
+        rows, rerun = retest_rows(e, labels, suites, repeat, tier, picks, force)
 
-        def answer(yes):
-            if yes:
-                self.app.start_or_queue(item)
-        lines = "\n".join(f"• {j.label} · {j.pack.label}: {n} {what} answers — {why}" for j, what, n, why in lost)
-        self.app.push_screen(ConfirmScreen(
-            "These packs would start over: their earlier answers ran with other questions or settings.\n\n"
-            f"{lines}\n\nThe earlier answers move to results/<model>/history/ (kept, but no longer counted). "
-            "To resume instead, pick No and put the setting back in models.toml.\n\nStart over?"), answer)
+        def go(choices):
+            if choices is None:
+                return
+            item = QueuedRun(sel, {k for k, c in choices.items() if c == "zero"},
+                             {k for k, c in choices.items() if c == "leave"},
+                             {(r.key, what) for r in rows if choices.get(r.key) == "zero" for what, _, _ in r.base.discards},
+                             e)
+            if not item.jobs:
+                self.notify(f"Nothing to run: {item.problem}.", severity="warning")
+                return
+            self.app.start_or_queue(item)
+        if rows:
+            self.app.push_screen(RetestScreen(rows, force, rerun), go)
+        else:
+            go({})
 
     def action_tune(self):
         e = self.app.engine
@@ -1540,14 +1544,16 @@ def run_title(tier, models, packs):
 
 class QueuedRun:
     """A run waiting its turn. It's planned again when it starts, so it picks up what the runs before
-    it finished; it never sets aside answers that weren't confirmed when it was queued."""
+    it finished; it never sets aside answers that weren't confirmed when it was queued. retest: the
+    job keys to ask again from zero; leave: the ones left out."""
     kind = "Run"
 
-    def __init__(self, sel, rerun, accepted, jobs, eng):
-        self.sel, self.rerun, self.accepted = sel, rerun, accepted
+    def __init__(self, sel, retest, leave, accepted, eng):
+        self.sel, self.retest, self.leave, self.accepted = sel, retest, leave, accepted
         self.queued = time.time()
+        self.jobs, self.problem = self.plan(eng)
+        jobs = self.jobs or []
         self.title = run_title(sel["tier"], {j.label for j in jobs}, {j.pack.label for j in jobs})
-        self.jobs = jobs
         self.est = eng.estimate_seconds(jobs)[0]
 
     def tests(self, eng):
@@ -1566,11 +1572,11 @@ class QueuedRun:
     def plan(self, eng):
         """(jobs, None), or (None, why it can't run) now."""
         sel = self.sel
-        jobs = eng.plan(sel["models"], sel["packs"], sel["repeat"], sel["tier"], sel["force"] or self.rerun,
-                        self.tests(eng))
+        jobs = [j for j in eng.plan(sel["models"], sel["packs"], sel["repeat"], sel["tier"], self.retest,
+                                    self.tests(eng)) if j.key not in self.leave]
         waiting = [j for j in jobs if j.status == "waiting"]
         if not waiting:
-            return None, "nothing left to run (earlier runs finished it)"
+            return None, "every pack left is already done"
         new = sorted({j.key for j in waiting for what, _, _ in j.discards if (j.key, what) not in self.accepted})
         if new:
             return None, (f"it would now start over {', '.join(new)}, which wasn't confirmed when it was queued. "
@@ -1621,6 +1627,140 @@ class ChoiceScreen(ModalScreen):
         self.dismiss(None)
 
 
+class RetestRow:
+    """A pack whose earlier answers a start could set aside. base: its job resuming; zero: from zero."""
+
+    def __init__(self, base, zero, finished, unfinished, choices, choice):
+        self.base, self.zero, self.finished, self.unfinished = base, zero, finished, unfinished
+        self.choices, self.choice = choices, choice
+        self.key = base.key
+
+    def earlier(self):
+        if self.base.discards:
+            return ", ".join(f"{n} {what}" for what, n, _ in self.base.discards)
+        parts = [f"{self.finished} finished"] * bool(self.finished) + \
+                [f"[yellow]{self.unfinished} unfinished[/yellow]"] * bool(self.unfinished)
+        return ", ".join(parts)
+
+    def happens(self):
+        b = self.base
+        if self.choice == "leave":
+            return "not run now"
+        if self.choice == "continue":
+            return "skipped: already done" if b.status == "done" else f"keeps {b.done}, asks the other {b.total - b.done}"
+        if b.discards:
+            why = "; ".join(sorted({why for _, _, why in b.discards}))
+            return f"[yellow]{why}[/yellow]; {sum(n for _, n, _ in b.discards)} → history/"
+        what = "picked tests asked again" if self.zero.redo else "asked again"
+        return f"{what}; {self.finished + self.unfinished} → history/"
+
+
+CHOICE_TEXT = {"zero": "[b red]Zero[/b red]", "continue": "[b green]Continue[/b green]", "leave": "[dim]Leave out[/dim]"}
+
+
+def retest_rows(eng, labels, packs, repeat, tier, picks, force):
+    """(rows, rerun): the packs Start would set earlier answers aside for, or could. rerun: Retest is
+    off but everything selected is done, so Start runs it again. A pack whose questions or settings
+    changed can't continue (Zero or Leave out); with Retest every pack with answers starts at Zero,
+    without it a pack that can resume continues."""
+    base = eng.plan(labels, packs, repeat, tier, False, picks)
+    zero = {j.key: j for j in eng.plan(labels, packs, repeat, tier, True, picks)}
+    rerun = not force and not any(j.status == "waiting" for j in base)
+    rows = []
+    for b in base:
+        z = zero[b.key]
+        if z.status != "waiting":
+            continue
+        finished, unfinished = eng.set_aside(z)
+        if b.discards:
+            rows.append(RetestRow(b, z, finished, unfinished, ["zero", "leave"], "zero"))
+        elif (force or rerun) and (finished or unfinished):
+            rows.append(RetestRow(b, z, finished, unfinished, ["zero", "continue"],
+                                  "zero" if force or b.status == "done" else "continue"))
+    return rows, rerun
+
+
+class RetestScreen(ModalScreen):
+    """Start's confirm when earlier answers would be set aside: a choice per model · pack. Dismisses
+    with {job key: "zero" | "continue" | "leave"}, or None on cancel."""
+    BINDINGS = [Binding("space", "toggle", "Toggle"), Binding("a", "all('zero')", "All from zero"),
+                Binding("c", "all('continue')", "All continue"), Binding("escape", "cancel", "Cancel")]
+
+    def __init__(self, rows, force, rerun):
+        super().__init__()
+        self.rows, self.force, self.rerun = rows, force, rerun
+
+    def compose(self) -> ComposeResult:
+        if self.force:
+            title = ("[b]Retest: choose what each pack does.[/b] Zero asks every question again; Continue keeps "
+                     "what's answered and asks the rest.")
+        elif self.rerun:
+            title = ("[b]Everything selected has results.[/b] Choose which packs to ask again (Zero) and which "
+                     "to keep (Continue).")
+        else:
+            title = ("[b]These packs can't continue:[/b] their earlier answers ran with other questions or "
+                     "settings. Start them from zero, or leave them out (put the setting back in models.toml "
+                     "to resume).")
+        with Vertical(id="dialog", classes="wide"):
+            yield Static(title)
+            yield DataTable(id="choices", cursor_type="row", zebra_stripes=True)
+            yield Static("[dim]space toggles the highlighted row · a all from zero · c all continue · "
+                         "Zero: earlier answers go to results/<model>/history/ (kept, not counted)[/dim]")
+            with Horizontal(classes="dialog-buttons"):
+                yield Button(self.start_label(), id="start", variant="primary")
+                yield Button("Cancel", id="cancel")
+
+    def on_mount(self):
+        t = self.query_one(DataTable)
+        t.add_column("Choice", key="choice", width=10)
+        t.add_column("Model", key="model")
+        t.add_column("Pack", key="pack")
+        t.add_column("Earlier answers", key="earlier")
+        t.add_column("What happens", key="happens")
+        for r in self.rows:
+            t.add_row(CHOICE_TEXT[r.choice], r.base.label, r.base.pack.label, r.earlier(), r.happens(), key=r.key)
+        self.update_start()
+        t.focus()
+
+    def redraw(self, r):
+        t = self.query_one(DataTable)
+        t.update_cell(r.key, "choice", CHOICE_TEXT[r.choice])
+        t.update_cell(r.key, "happens", r.happens(), update_width=True)
+        self.update_start()
+
+    def start_label(self):
+        n = collections.Counter(r.choice for r in self.rows)
+        parts = [f"{n[c]} {w}" for c, w in (("zero", "from zero"), ("continue", "continue"), ("leave", "left out"))
+                 if n[c]]
+        return f"Start ({', '.join(parts)})"
+
+    def update_start(self):
+        button = self.query_one("#start", Button)
+        button.label = self.start_label()
+        button.refresh(layout=True)
+
+    def action_toggle(self):
+        t = self.query_one(DataTable)
+        if not self.rows:
+            return
+        r = self.rows[t.cursor_row]
+        r.choice = r.choices[(r.choices.index(r.choice) + 1) % len(r.choices)]
+        self.redraw(r)
+
+    def action_all(self, choice):
+        for r in self.rows:
+            if choice in r.choices:
+                r.choice = choice
+                self.redraw(r)
+
+    def action_cancel(self):
+        self.dismiss(None)
+
+    @on(Button.Pressed)
+    def answer(self, event):
+        self.dismiss({r.key: r.choice for r in self.rows} if event.button.id == "start" else None)
+
+
 class ConfirmScreen(ModalScreen):
     def __init__(self, question):
         super().__init__()
@@ -2099,9 +2239,10 @@ class RunScreen(Screen):
             self.req_started, self.req_tokens, self.req_first = time.time(), 0, None
             job = d["job"]
             rep_note = f" · repeat {d['repeat'] + 1}/{job.repeat}" if job.repeat > 1 else ""
-            self.current = (f"[b]{job.label}[/b] · {job.pack.label} · test {job.done + 1}/{job.total}{rep_note}"
+            self.current = (f"[b]{rich_escape(job.label)}[/b] · {rich_escape(job.pack.label)} · test "
+                            f"{job.done + 1}/{job.total}{rep_note}"
                             f"{'  [magenta]🖼 image[/magenta]' if d['has_image'] else ''}\n"
-                            f"{diff_badge(d.get('difficulty', 'unrated'))}  {d['test']}")
+                            f"{diff_badge(d.get('difficulty', 'unrated'))}  {rich_escape(d['test'])}")
         elif kind == "request_done":
             if self.watched(kind, d):
                 self.flush_streams()
@@ -2111,12 +2252,12 @@ class RunScreen(Screen):
             # follow new rows unless you're browsing an older one
             at_end = not recent.has_focus or recent.cursor_row >= recent.row_count - 1
             calls = f"  [cyan]→ {r['tool_calls'][0]['name']}()[/cyan]" if r.get("tool_calls") else ""
-            row_key = recent.add_row(mark, diff_badge(r.get("difficulty", "unrated")), r["description"][:50], job.label,
+            row_key = recent.add_row(mark, diff_badge(r.get("difficulty", "unrated")), rich_escape(r["description"][:50]), job.label,
                            str(r.get("completion_tokens") or ""),
                            (f"{r['ttft_s']:.1f}" + (" ⟲" if r.get("cached_tokens") else ""))
                            if r.get("ttft_s") is not None else "",
                            f"{r['gen_tps']:.0f}" if r.get("gen_tps") else "", f"{r['total_s']:.0f}",
-                           r["reason"][:90] + calls
+                           rich_escape(r["reason"][:90]) + calls
                            + (f"  [red]copy of repeat {r['identical_to_repeat'] + 1}[/red]"
                               if r.get("identical_to_repeat") is not None else ""))
             self.stats.append(r)
@@ -2224,7 +2365,7 @@ class RunScreen(Screen):
             self.event("[bold yellow]REUSE[/bold yellow] " + msg)
             self.notify(msg, severity="warning", timeout=20)
         elif kind == "identical_answer":
-            msg = (f"{d['label']} · {d['pack']} · {d['test']}: repeat {d['repeat'] + 1} is an exact copy of repeat "
+            msg = (f"{d['label']} · {d['pack']} · {rich_escape(d['test'])}: repeat {d['repeat'] + 1} is an exact copy of repeat "
                    f"{d['same'] + 1}, so the server may be returning cached answers rather than fresh ones.")
             self.event("[bold red]COPY[/bold red] " + msg)
             self.notify(msg, severity="error", timeout=20)
@@ -2571,7 +2712,7 @@ class ResultsScreen(Screen):
         # compare.failures lists the failed rows in order, so they line up with fail_rows
         self.fail_rows = [r for r in rows if not r["ok"]]
         for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
-            f.add_row(model, diff_badge(level), test, rich_escape(reason[:130]), key=str(i))
+            f.add_row(model, diff_badge(level), rich_escape(test), rich_escape(reason[:130]), key=str(i))
         self.fill_per_question()
         self.fill_pta()
         self.fill_readiness(results_dir)
@@ -2955,7 +3096,8 @@ class EvalsApp(App):
     #reasoning { color: $text-muted; }
     StreamView { height: 1fr; border: none; padding: 0; }
     #tabs { height: 12; }
-    AddModelScreen, ConfirmScreen { align: center middle; }
+    AddModelScreen, ConfirmScreen, RetestScreen { align: center middle; }
+    RetestScreen #start { width: auto; }
     SelectionList > .selection-list--button, SelectionList > .selection-list--button-highlighted {
         color: $panel; background: $panel;
     }
@@ -3094,8 +3236,6 @@ class EvalsApp(App):
             if problem:
                 self.notify(f"Can't start {item.title}: {problem}", severity="warning", timeout=15)
                 return
-            if item.kind == "Run" and item.rerun:
-                self.notify("Everything selected had results; running it again.")
             self.add_session(screen)
             return
         run = self.active_session

{tuieval-0.2.0.dev72 → tuieval-0.2.0.dev75}/tests/test_tuieval.py

@@ -411,6 +411,160 @@ class PickTests(unittest.TestCase):
         self.assertEqual(len(self.rows("other")), 1)     # no pick: the Screen sample (screen = 1)
 
 
+class Retest(unittest.TestCase):
+    """Retest starts from zero (an unfinished run's answers too), keeps every replaced answer in
+    history/ (smoke too), and the TUI asks per pack before anything moves."""
+
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.ws = os.path.join(self.tmp.name, "ws")
+        tuieval(self.tmp.name, "init", self.ws)
+        for name in ("apps", "other"):
+            d = os.path.join(self.ws, "packs", name)
+            os.makedirs(d)
+            write(os.path.join(d, "pack.toml"), f'label = "{name}"\nscreen = 4\n[certify]\nrepeat = 1\n'
+                                                '[gate]\nmin_accuracy = 0.1\n')
+            write(os.path.join(d, "tests.yaml"), "".join(
+                f"- {{id: {t}, input: {name} question {t}, expected: 1, reference: 'ANSWER: 1', wrong: ['ANSWER: 2'], "
+                "difficulty: easy}\n" for t in "abcd"))
+        self.port = free_port()
+        with open(os.path.join(self.ws, "models.toml"), "a") as f:
+            f.write(f'\n[servers.m]\nurl = "http://127.0.0.1:{self.port}"\n')
+        self.mock = Mock(os.path.join(self.ws, "packs"), "oracle", port=self.port)
+        tuieval(self.ws, "add", "mock", "--server", "m", "--label", "m")
+        tuieval(self.ws, "run", "--tier", "certify")
+        self.mock.stop()
+        self.mock = Mock(os.path.join(self.ws, "packs"), "wrong", port=self.port)   # retested answers fail
+
+    def tearDown(self):
+        self.mock.stop()
+        self.tmp.cleanup()
+
+    def path(self, pack, smoke=False):
+        return os.path.join(self.ws, "results", *(["smoke"] if smoke else []), "m", f"{pack}.json")
+
+    def passes(self, pack):
+        return {r["test"]: r["pass"] for r in json.loads(read(self.path(pack)))["results"]}
+
+    def unfinished(self, pack, tests):
+        """Turn a pack's finished answers for these tests into an unfinished run of the same settings."""
+        from tuieval import engine
+        e = engine.Engine(root=self.ws)
+        j = e.plan(["m"], [pack], None, "certify", False, {})[0]
+        rows = [r for r in json.loads(read(j.out_path))["results"] if r["test"] in tests]
+        write(j.out_path + ".partial.jsonl", "".join(json.dumps(l) + "\n" for l in [e._partial_header(j)] + rows))
+        os.remove(j.out_path)
+
+    def history(self, smoke=False):
+        d = os.path.join(self.ws, "results", *(["smoke"] if smoke else []), "m", "history")
+        return sorted(os.listdir(d)) if os.path.isdir(d) else []
+
+    def test_cli_retest_starts_from_zero(self):
+        self.unfinished("apps", "ab")
+        out = tuieval(self.ws, "run", "--tier", "certify", "--packs", "apps", "--retest").stdout
+        self.assertIn("m/apps: retest from zero; its 2 unfinished answers move to history/", out)
+        self.assertNotIn("resuming", out)
+        self.assertEqual(self.passes("apps"), dict.fromkeys("abcd", False))        # all asked again
+        self.assertTrue(any(f.endswith(".partial.jsonl") for f in self.history()))
+
+    def test_picked_retest_keeps_the_rest_of_an_unfinished_run(self):
+        self.unfinished("apps", "abc")
+        tuieval(self.ws, "run", "--tier", "certify", "--tests", "apps:b", "--retest")
+        partial = self.path("apps") + ".partial.jsonl"
+        self.assertFalse(os.path.exists(partial))
+        self.assertEqual(self.passes("apps"), {"a": True, "b": False, "c": True})   # only b asked again
+        self.assertTrue(any(f.endswith(".partial.jsonl") for f in self.history()))   # a copy, b included
+
+    def test_smoke_retest_keeps_history(self):
+        tuieval(self.ws, "run", "--tier", "smoke", "--packs", "apps")
+        tuieval(self.ws, "run", "--tier", "smoke", "--packs", "apps", "--retest")
+        self.assertEqual(len(self.history(smoke=True)), 1)
+        self.assertEqual(tuieval(self.ws, "list").returncode, 0)
+
+    def tui(self, steps):
+        """Drive Setup with apps and other ticked for model m (Certify); steps runs against the screen."""
+        code = textwrap.dedent("""
+            import asyncio, json, time
+            from tuieval.tui import EvalsApp, RetestScreen, RunScreen
+            from textual.widgets import SelectionList, Checkbox
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(160, 50)) as pilot:
+                    await pilot.pause()
+                    s = app.screen
+                    for p in ("apps", "other"):
+                        s.query_one("#suites", SelectionList).select(p)
+                    s.selected_models = {"m"}
+                    s.refresh_models()
+                    s.query_one("#tier-certify").value = True
+                    out = {}
+            %s
+                    deadline = time.time() + 120
+                    while time.time() < deadline and (app.queue or (isinstance(app.active_session, RunScreen)
+                                                                    and app.active_session.running)):
+                        await pilot.pause(0.2)
+                    out["sessions"] = len(app.sessions)
+                    print(json.dumps(out))
+            asyncio.run(go())
+        """) % textwrap.indent(textwrap.dedent(steps), " " * 8)
+        p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=300,
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""))
+        out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
+        self.assertIsInstance(out, dict, out)
+        return out
+
+    def test_tui_asks_per_pack(self):
+        self.unfinished("other", "ab")
+        out = self.tui("""
+            s.query_one("#force", Checkbox).value = True
+            await pilot.pause()
+            s.action_start()
+            await pilot.pause()
+            d = app.screen
+            out["asked"] = isinstance(d, RetestScreen)
+            out["rows"] = [(r.key, r.choice, r.finished, r.unfinished) for r in d.rows]
+            await pilot.press("down", "space")           # other: Continue
+            out["button"] = str(d.query_one("#start").label)
+            await pilot.press("escape")                  # cancel: nothing runs
+            await pilot.pause()
+            out["after_cancel"] = len(app.sessions)
+            s.action_start()
+            await pilot.pause()
+            await pilot.press("down", "space")
+            await pilot.pause()
+            app.screen.query_one("#start").press()
+            await pilot.pause()
+        """)
+        self.assertTrue(out["asked"])
+        self.assertEqual(out["rows"], [["m/apps", "zero", 4, 0], ["m/other", "zero", 0, 2]])
+        self.assertEqual(out["button"], "Start (1 from zero, 1 continue)")
+        self.assertEqual(out["after_cancel"], 0)
+        self.assertEqual(out["sessions"], 1)
+        self.assertEqual(self.passes("apps"), dict.fromkeys("abcd", False))                  # from zero
+        self.assertEqual(self.passes("other"), {"a": True, "b": True, "c": False, "d": False})   # continued
+
+    def test_tui_everything_done_or_settings_changed(self):
+        models = os.path.join(self.ws, "models.toml")
+        write(models, read(models).replace("temperature = 0.7", "temperature = 0.6"))
+        out = self.tui("""
+            s.query_one("#suites", SelectionList).deselect("other")
+            await pilot.pause()
+            s.action_start()
+            await pilot.pause()
+            d = app.screen
+            out["rows"] = [(r.key, r.choices, r.choice) for r in d.rows]
+            await pilot.press("space")                   # Leave out
+            await pilot.pause()
+            out["happens"] = d.rows[0].happens()
+            d.query_one("#start").press()
+            await pilot.pause()
+        """)
+        self.assertEqual(out["rows"], [["m/apps", ["zero", "leave"], "zero"]])
+        self.assertEqual(out["happens"], "not run now")
+        self.assertEqual(out["sessions"], 0)                       # nothing ran
+        self.assertEqual(self.passes("apps"), dict.fromkeys("abcd", True))
+
+
 class Parallel(unittest.TestCase):
     """Several models at a time (parallel_models / --parallel): each runs in its own lane on its own
     port, answers note the models they ran alongside, and the scheduler keeps the memory and
@@ -581,6 +735,56 @@ class Parallel(unittest.TestCase):
         self.assertEqual(sum(r["pass"] for r in self.results("b")["results"]), 6)
 
 
+class MarkupInText(unittest.TestCase):
+    """Brackets in a description or a grader's reason are shown as text: an unmatched
+    "[" once crashed the run screen (it swallowed the "[dim]" of the line after it)."""
+
+    def test_run_screen_shows_brackets(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            ws = os.path.join(tmp, "ws")
+            tuieval(tmp, "init", ws)
+            d = os.path.join(ws, "packs", "br")
+            os.makedirs(d)
+            write(os.path.join(d, "pack.toml"), 'label = "Brackets"\n[certify]\nrepeat = 1\n[gate]\nmin_accuracy = 0.1\n')
+            write(os.path.join(d, "tests.yaml"),
+                  "- {id: a, description: 'notional in [t - 10 s, t), excluding ties', input: q a, expected: 1, "
+                  "reference: 'ANSWER: 1', wrong: ['ANSWER: [2'], difficulty: hard}\n"
+                  "- {id: b, description: 'window [a, b) then [/dim] text', input: q b, expected: 1, "
+                  "reference: 'ANSWER: 1', wrong: ['ANSWER: [2'], difficulty: hard}\n")
+            mock = [sys.executable, os.path.join(HERE, "mock_server.py"), "--port", "{port}", "--packs",
+                    os.path.join(ws, "packs"), "--model", "{served_name}", "--mode", "wrong", "--delay", "0.2"]
+            with open(os.path.join(ws, "models.toml"), "a") as f:
+                f.write(f'\n[servers.mk]\ncmd = {json.dumps(mock)}\nport = {free_port()}\n'
+                        '\n[[models]]\nlabel = "m"\nserver = "mk"\nmodel = "m"\n')
+            code = textwrap.dedent("""
+                import asyncio, json, time
+                from tuieval.tui import EvalsApp, RunScreen
+                from textual.widgets import SelectionList
+                async def go():
+                    app = EvalsApp({})
+                    async with app.run_test(size=(160, 50)) as pilot:
+                        await pilot.pause()
+                        s = app.screen
+                        s.query_one("#suites", SelectionList).select("br")
+                        s.selected_models = {"m"}
+                        s.refresh_models()
+                        s.query_one("#tier-certify").value = True
+                        await pilot.pause()
+                        s.action_start()
+                        deadline = time.time() + 240
+                        while time.time() < deadline:
+                            await pilot.pause(0.2)
+                            if isinstance(app.screen, RunScreen) and not app.screen.running:
+                                break
+                        print(json.dumps([(j.key, j.status, j.done) for j in app.screen.jobs]))
+                asyncio.run(go())
+            """)
+            p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=480,
+                               env=dict(os.environ, TUIEVAL_HOME=ws, TUIEVAL_DETECT_PORTS=""))
+            self.assertNotIn("MarkupError", p.stderr + p.stdout)
+            self.assertEqual(json.loads(p.stdout.strip().splitlines()[-1]), [["m/br", "done", 2]], p.stderr[-2000:])
+
+
 class AddToRun(unittest.TestCase):
     """More evals while a run is going: added to it (they run after its other jobs) or queued after it."""