tuieval 0.2.0.dev116__tar.gz → 0.2.0.dev119__tar.gz

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev119}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Setup opens with no models ticked: at launch, after New run, and after Back from a run or tune. Packs and options are still remembered.
+- Tuning: a server that refuses the `ignore_eos` request field is asked again without it, so strict servers can be tuned; the profile notes that workload answers may end early.
 - Fix: a server that crashes mid-request (the stream ends with no finish reason) was graded as an empty or cut-short answer; it's a server error now, never counted against the model.
 - Dates: answers record `answered_at` (older ones read it from their log name); tests get an added date (an `added` field, or the day tuieval first saw them, in `test-dates.json`). Shown in the answer view, Pick tests, Per question, Failures and Production readiness.
 - Copy reasoning: `y` in the answer view copies its full reasoning, `a` the answer; `y` on the run screen copies the reasoning streamed so far. Uses pbcopy, wl-copy, xclip, xsel or clip, else the terminal (OSC 52).

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev119}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev116
+Version: 0.2.0.dev119
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev119}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev116'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev116')
+__version__ = version = '0.2.0.dev119'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev119')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev119}/src/tuieval/tui.py

@@ -751,11 +751,17 @@ class SetupScreen(Screen):
         self.refresh_packs()
         self.refresh_models()
         state = self.app.load_state()
-        self.apply_selection(state, quiet=True)
-        if not state:
+        if state:   # packs and options carry over; models start unticked every time
+            self.apply_selection({k: v for k, v in state.items() if k != "models"}, quiet=True)
+        else:
             self.first_time_defaults()
         self.query_one("#suites", SelectionList).focus()
 
+    def clear_models(self):
+        """Untick every model (Setup shown again after a run: each run picks its models afresh)."""
+        self.selected_models = set()
+        self.refresh_models()
+
     def first_time_defaults(self):
         """With nothing chosen before: tick the only pack and the only local model, and start on
         Smoke until something has run (a 3-question check of the setup)."""
@@ -1155,7 +1161,7 @@ class SetupScreen(Screen):
             reasons = sorted({j.note for j in jobs if j.status == "skipped"})
             self.notify(f"Nothing selected can run: {'; '.join(reasons)}", severity="warning")
             return
-        self.app.save_state(sel)
+        self.app.save_state({k: v for k, v in sel.items() if k != "models"})   # models aren't remembered
         # Packs whose earlier answers a start would set aside (Retest, everything already done, or
         # other questions or settings): the user picks per pack before anything moves.
         rows, rerun = retest_rows(e, labels, suites, repeat, tier, picks, force)
@@ -1349,7 +1355,7 @@ class TuneScreen(Screen):
     def action_back(self):
         if self.running:
             self.notify("Tuning keeps going; the header shows its progress. Press w to come back.")
-        self.app.back_to_setup()
+        self.app.back_to_setup(clear_models=True)
 
 
 # ------------------------------------------------------------------ add model
@@ -2825,7 +2831,7 @@ class RunScreen(Screen):
         if self.running:
             self.notify("Pick models and packs, then Start: add them to this run or queue them after it.",
                         timeout=8)
-        self.app.back_to_setup()  # Setup still holds the last selection
+        self.app.back_to_setup(clear_models=True)  # packs and options stay; models are picked afresh
         self.app.screen.query_one("#models").focus()
 
     @on(Button.Pressed, "#done-runs")
@@ -2835,7 +2841,7 @@ class RunScreen(Screen):
     def action_back(self):
         if self.running:
             self.notify("The run keeps going; the header shows its progress. Press w to come back.")
-        self.app.back_to_setup()
+        self.app.back_to_setup(clear_models=True)
 
 
 # ------------------------------------------------------------------ results
@@ -3789,9 +3795,12 @@ class EvalsApp(App):
         return (f"\n\nThe {len(self.queue)} queued item(s) are dropped (the queue isn't kept after quitting): "
                 + "; ".join(i.title for i in self.queue) + ".")
 
-    def back_to_setup(self):
+    def back_to_setup(self, clear_models=False):
+        """clear_models: Setup is being shown for a new pick (New run, Back), so no model stays ticked."""
         while len(self.screen_stack) > 2 and not isinstance(self.screen, SetupScreen):
             self.pop_screen()
+        if clear_models and isinstance(self.screen, SetupScreen):
+            self.screen.clear_models()
 
     def show_session(self, screen):
         if screen is None or screen is self.screen:

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev119}/src/tuieval/tune.py

@@ -169,7 +169,37 @@ def _body(m, sampling, msgs, max_tokens=GEN_TOKENS, request=None):
             **(request or {})}
 
 
+def _rejects_ignore_eos(res):
+    """True if the server refused the request because it doesn't accept the ignore_eos field."""
+    err = res.get("error") or ""
+    return err.startswith("server returned HTTP 4") and "ignore_eos" in err
+
+
 def _request(eng, base_url, body, name, emit, limit):
+    """One streamed request with live progress and a wall-clock limit. A server that refuses
+    ignore_eos is asked again without it, and later requests leave it out (see no_ignore_eos)."""
+    key = (base_url, body.get("model"))
+    rejected = no_ignore_eos(eng)
+    if key in rejected:
+        body = {k: v for k, v in body.items() if k != "ignore_eos"}
+    res = _stream_request(eng, base_url, body, name, emit, limit)
+    if body.get("ignore_eos") and _rejects_ignore_eos(res):
+        rejected.add(key)
+        emit("tune_step", message="    the server doesn't accept ignore_eos: answers may end early "
+                                  "(speeds are per token, so still comparable)")
+        res = _stream_request(eng, base_url, {k: v for k, v in body.items() if k != "ignore_eos"},
+                              name, emit, limit)
+    return res
+
+
+def no_ignore_eos(eng):
+    """(base_url, served model) pairs whose server refused ignore_eos in this tuning session."""
+    if not hasattr(eng, "_tune_no_ignore_eos"):
+        eng._tune_no_ignore_eos = set()
+    return eng._tune_no_ignore_eos
+
+
+def _stream_request(eng, base_url, body, name, emit, limit):
     """One streamed request with live progress (every PROGRESS_EVERY_S: waiting for the first
     token, or ~tokens so far and tok/s) and a wall-clock limit. Returns client.stream_chat's result."""
     st, t0 = client.Stream(), time.time()
@@ -681,6 +711,9 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         if warm_info:
             notes.insert(0, f"warm start from {warm_info['from']}: inherited "
                             f"{', '.join(warm_info['inherited']) or 'nothing'}")
+    if any(model == m["served_name"] for _, model in no_ignore_eos(eng)):
+        notes.insert(0, f"the server doesn't accept ignore_eos, so workload answers could end before "
+                        f"{gen_tokens} tokens (speeds are per token)")
     if best_r.facts.get("extra_swap_mb") and best_r is not base:
         warnings.append(f"the chosen flags use more memory than the defaults: macOS swapped "
                         f"~{best_r.facts['extra_swap_mb']} MB more of other apps' memory; close apps during runs, "

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev119}/tests/test_tuieval.py

@@ -2016,6 +2016,32 @@ class FirstRun(unittest.TestCase):
                              env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""), timeout=60).stdout.strip()
         self.assertEqual(out, "['first'] ['my-model'] smoke False")   # tuning shows once something has run
 
+    def test_setup_never_starts_with_models_ticked(self):
+        tuieval(self.ws, "add", "my-model", "--server", "local")
+        os.makedirs(os.path.join(self.ws, "logs"), exist_ok=True)
+        write(os.path.join(self.ws, "logs", "tui_state.json"),
+              json.dumps({"models": ["my-model"], "packs": ["first"], "tier": "certify", "repeat": 2}))
+        code = textwrap.dedent("""
+            import asyncio
+            from tuieval.tui import EvalsApp
+            from textual.widgets import SelectionList
+            async def go():
+                app = EvalsApp({})
+                async with app.run_test(size=(140, 40)) as pilot:
+                    await pilot.pause()
+                    s = app.screen
+                    print(s.query_one("#suites", SelectionList).selected, sorted(s.selected_models), s.tier())
+                    s.selected_models = {"my-model"}
+                    s.refresh_models()
+                    app.back_to_setup(clear_models=True)   # what New run and Back do
+                    await pilot.pause()
+                    print(s.query_one("#suites", SelectionList).selected, sorted(s.selected_models))
+            asyncio.run(go())
+        """)
+        out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True,
+                             env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""), timeout=60).stdout
+        self.assertEqual(out.strip().splitlines(), ["['first'] [] certify", "['first'] []"])
+
     def test_trimmed_names_keep_both_ends(self):
         from tuieval.tui import fit_first_column, mid_trim, trim_names
         name = "some-author--some-long-model-name-q4_k_m-vllm"
@@ -2512,6 +2538,37 @@ class WarmTune(unittest.TestCase):
         self.assertEqual(out.score("projected"), 66.0)
         self.assertEqual(out.score("total"), 10.0)
 
+    def test_server_refusing_ignore_eos_is_asked_without_it(self):
+        sent = []
+
+        def server(eng, base_url, body, *a):   # refuses unknown fields, like a strict server
+            sent.append(dict(body))
+            if "ignore_eos" in body:
+                return {"error": 'server returned HTTP 400: {"error":{"message":"unsupported request field: '
+                                 'ignore_eos"}}', "answer": "", "reasoning": ""}
+            return {"error": None, "answer": "ok", "reasoning": ""}
+        orig = self.tune._stream_request
+        self.tune._stream_request = server
+        self.addCleanup(setattr, self.tune, "_stream_request", orig)
+        body = {"model": "new", "messages": [], "ignore_eos": True}
+        res = self.tune._request(self.eng, "http://x", body, "p", lambda *a, **k: None, 60)
+        self.assertIsNone(res["error"])
+        self.assertEqual(["ignore_eos" in b for b in sent], [True, False])   # retried without it
+        self.tune._request(self.eng, "http://x", body, "p", lambda *a, **k: None, 60)
+        self.assertEqual(len(sent), 3)   # later requests leave it out at once
+        self.assertNotIn("ignore_eos", sent[-1])
+        self.assertIn(("http://x", "new"), self.tune.no_ignore_eos(self.eng))
+
+    def test_other_request_errors_are_not_retried(self):
+        sent = []
+        orig = self.tune._stream_request
+        self.tune._stream_request = lambda eng, url, body, *a: sent.append(body) or {
+            "error": "server returned HTTP 400: bad messages", "answer": "", "reasoning": ""}
+        self.addCleanup(setattr, self.tune, "_stream_request", orig)
+        res = self.tune._request(self.eng, "http://x", {"model": "new", "ignore_eos": True}, "p",
+                                 lambda *a, **k: None, 60)
+        self.assertEqual((len(sent), res["error"]), (1, "server returned HTTP 400: bad messages"))
+
     def test_family_ignores_the_mtp_layer(self):
         # one GGUF lists KV heads per layer, none on the MTP (last) layer; the other gives one number
         d = self.tmp.name