tuieval 0.2.0.dev118__tar.gz → 0.2.0.dev119__tar.gz

{tuieval-0.2.0.dev118 → tuieval-0.2.0.dev119}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Setup opens with no models ticked: at launch, after New run, and after Back from a run or tune. Packs and options are still remembered.
 - Tuning: a server that refuses the `ignore_eos` request field is asked again without it, so strict servers can be tuned; the profile notes that workload answers may end early.
 - Fix: a server that crashes mid-request (the stream ends with no finish reason) was graded as an empty or cut-short answer; it's a server error now, never counted against the model.
 - Dates: answers record `answered_at` (older ones read it from their log name); tests get an added date (an `added` field, or the day tuieval first saw them, in `test-dates.json`). Shown in the answer view, Pick tests, Per question, Failures and Production readiness.

{tuieval-0.2.0.dev118 → tuieval-0.2.0.dev119}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev118
+Version: 0.2.0.dev119
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev118 → tuieval-0.2.0.dev119}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev118'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev118')
+__version__ = version = '0.2.0.dev119'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev119')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev118 → tuieval-0.2.0.dev119}/src/tuieval/tui.py

@@ -751,11 +751,17 @@ class SetupScreen(Screen):
         self.refresh_packs()
         self.refresh_models()
         state = self.app.load_state()
-        self.apply_selection(state, quiet=True)
-        if not state:
+        if state:   # packs and options carry over; models start unticked every time
+            self.apply_selection({k: v for k, v in state.items() if k != "models"}, quiet=True)
+        else:
             self.first_time_defaults()
         self.query_one("#suites", SelectionList).focus()
 
+    def clear_models(self):
+        """Untick every model (Setup shown again after a run: each run picks its models afresh)."""
+        self.selected_models = set()
+        self.refresh_models()
+
     def first_time_defaults(self):
         """With nothing chosen before: tick the only pack and the only local model, and start on
         Smoke until something has run (a 3-question check of the setup)."""
@@ -1155,7 +1161,7 @@ class SetupScreen(Screen):
             reasons = sorted({j.note for j in jobs if j.status == "skipped"})
             self.notify(f"Nothing selected can run: {'; '.join(reasons)}", severity="warning")
             return
-        self.app.save_state(sel)
+        self.app.save_state({k: v for k, v in sel.items() if k != "models"})   # models aren't remembered
         # Packs whose earlier answers a start would set aside (Retest, everything already done, or
         # other questions or settings): the user picks per pack before anything moves.
         rows, rerun = retest_rows(e, labels, suites, repeat, tier, picks, force)
@@ -1349,7 +1355,7 @@ class TuneScreen(Screen):
     def action_back(self):
         if self.running:
             self.notify("Tuning keeps going; the header shows its progress. Press w to come back.")
-        self.app.back_to_setup()
+        self.app.back_to_setup(clear_models=True)
 
 
 # ------------------------------------------------------------------ add model
@@ -2825,7 +2831,7 @@ class RunScreen(Screen):
         if self.running:
             self.notify("Pick models and packs, then Start: add them to this run or queue them after it.",
                         timeout=8)
-        self.app.back_to_setup()  # Setup still holds the last selection
+        self.app.back_to_setup(clear_models=True)  # packs and options stay; models are picked afresh
         self.app.screen.query_one("#models").focus()
 
     @on(Button.Pressed, "#done-runs")
@@ -2835,7 +2841,7 @@ class RunScreen(Screen):
     def action_back(self):
         if self.running:
             self.notify("The run keeps going; the header shows its progress. Press w to come back.")
-        self.app.back_to_setup()
+        self.app.back_to_setup(clear_models=True)
 
 
 # ------------------------------------------------------------------ results
@@ -3789,9 +3795,12 @@ class EvalsApp(App):
         return (f"\n\nThe {len(self.queue)} queued item(s) are dropped (the queue isn't kept after quitting): "
                 + "; ".join(i.title for i in self.queue) + ".")
 
-    def back_to_setup(self):
+    def back_to_setup(self, clear_models=False):
+        """clear_models: Setup is being shown for a new pick (New run, Back), so no model stays ticked."""
         while len(self.screen_stack) > 2 and not isinstance(self.screen, SetupScreen):
             self.pop_screen()
+        if clear_models and isinstance(self.screen, SetupScreen):
+            self.screen.clear_models()
 
     def show_session(self, screen):
         if screen is None or screen is self.screen:

{tuieval-0.2.0.dev118 → tuieval-0.2.0.dev119}/tests/test_tuieval.py

@@ -2016,6 +2016,32 @@ class FirstRun(unittest.TestCase):
                              env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""), timeout=60).stdout.strip()
         self.assertEqual(out, "['first'] ['my-model'] smoke False")   # tuning shows once something has run
 
+    def test_setup_never_starts_with_models_ticked(self):
+        tuieval(self.ws, "add", "my-model", "--server", "local")
+        os.makedirs(os.path.join(self.ws, "logs"), exist_ok=True)
+        write(os.path.join(self.ws, "logs", "tui_state.json"),
+              json.dumps({"models": ["my-model"], "packs": ["first"], "tier": "certify", "repeat": 2}))
+        code = textwrap.dedent("""
+            import asyncio
+            from tuieval.tui import EvalsApp
+            from textual.widgets import SelectionList
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(140, 40)) as pilot:
+                    await pilot.pause()
+                    s = app.screen
+                    print(s.query_one("#suites", SelectionList).selected, sorted(s.selected_models), s.tier())
+                    s.selected_models = {"my-model"}
+                    s.refresh_models()
+                    app.back_to_setup(clear_models=True)   # what New run and Back do
+                    await pilot.pause()
+                    print(s.query_one("#suites", SelectionList).selected, sorted(s.selected_models))
+            asyncio.run(go())
+        """)
+        out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True,
+                             env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""), timeout=60).stdout
+        self.assertEqual(out.strip().splitlines(), ["['first'] [] certify", "['first'] []"])
+
     def test_trimmed_names_keep_both_ends(self):
         from tuieval.tui import fit_first_column, mid_trim, trim_names
         name = "some-author--some-long-model-name-q4_k_m-vllm"