tuieval 0.2.0.dev116__tar.gz → 0.2.0.dev118__tar.gz

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev118}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Tuning: a server that refuses the `ignore_eos` request field is asked again without it, so strict servers can be tuned; the profile notes that workload answers may end early.
 - Fix: a server that crashes mid-request (the stream ends with no finish reason) was graded as an empty or cut-short answer; it's a server error now, never counted against the model.
 - Dates: answers record `answered_at` (older ones read it from their log name); tests get an added date (an `added` field, or the day tuieval first saw them, in `test-dates.json`). Shown in the answer view, Pick tests, Per question, Failures and Production readiness.
 - Copy reasoning: `y` in the answer view copies its full reasoning, `a` the answer; `y` on the run screen copies the reasoning streamed so far. Uses pbcopy, wl-copy, xclip, xsel or clip, else the terminal (OSC 52).

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev118}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev116
+Version: 0.2.0.dev118
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev118}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev116'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev116')
+__version__ = version = '0.2.0.dev118'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev118')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev118}/src/tuieval/tune.py

@@ -169,7 +169,37 @@ def _body(m, sampling, msgs, max_tokens=GEN_TOKENS, request=None):
             **(request or {})}
 
 
+def _rejects_ignore_eos(res):
+    """True if the server refused the request because it doesn't accept the ignore_eos field."""
+    err = res.get("error") or ""
+    return err.startswith("server returned HTTP 4") and "ignore_eos" in err
+
+
 def _request(eng, base_url, body, name, emit, limit):
+    """One streamed request with live progress and a wall-clock limit. A server that refuses
+    ignore_eos is asked again without it, and later requests leave it out (see no_ignore_eos)."""
+    key = (base_url, body.get("model"))
+    rejected = no_ignore_eos(eng)
+    if key in rejected:
+        body = {k: v for k, v in body.items() if k != "ignore_eos"}
+    res = _stream_request(eng, base_url, body, name, emit, limit)
+    if body.get("ignore_eos") and _rejects_ignore_eos(res):
+        rejected.add(key)
+        emit("tune_step", message="    the server doesn't accept ignore_eos: answers may end early "
+                                  "(speeds are per token, so still comparable)")
+        res = _stream_request(eng, base_url, {k: v for k, v in body.items() if k != "ignore_eos"},
+                              name, emit, limit)
+    return res
+
+
+def no_ignore_eos(eng):
+    """(base_url, served model) pairs whose server refused ignore_eos in this tuning session."""
+    if not hasattr(eng, "_tune_no_ignore_eos"):
+        eng._tune_no_ignore_eos = set()
+    return eng._tune_no_ignore_eos
+
+
+def _stream_request(eng, base_url, body, name, emit, limit):
     """One streamed request with live progress (every PROGRESS_EVERY_S: waiting for the first
     token, or ~tokens so far and tok/s) and a wall-clock limit. Returns client.stream_chat's result."""
     st, t0 = client.Stream(), time.time()
@@ -681,6 +711,9 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         if warm_info:
             notes.insert(0, f"warm start from {warm_info['from']}: inherited "
                             f"{', '.join(warm_info['inherited']) or 'nothing'}")
+    if any(model == m["served_name"] for _, model in no_ignore_eos(eng)):
+        notes.insert(0, f"the server doesn't accept ignore_eos, so workload answers could end before "
+                        f"{gen_tokens} tokens (speeds are per token)")
     if best_r.facts.get("extra_swap_mb") and best_r is not base:
         warnings.append(f"the chosen flags use more memory than the defaults: macOS swapped "
                         f"~{best_r.facts['extra_swap_mb']} MB more of other apps' memory; close apps during runs, "

{tuieval-0.2.0.dev116 → tuieval-0.2.0.dev118}/tests/test_tuieval.py

@@ -2512,6 +2512,37 @@ class WarmTune(unittest.TestCase):
         self.assertEqual(out.score("projected"), 66.0)
         self.assertEqual(out.score("total"), 10.0)
 
+    def test_server_refusing_ignore_eos_is_asked_without_it(self):
+        sent = []
+
+        def server(eng, base_url, body, *a):   # refuses unknown fields, like a strict server
+            sent.append(dict(body))
+            if "ignore_eos" in body:
+                return {"error": 'server returned HTTP 400: {"error":{"message":"unsupported request field: '
+                                 'ignore_eos"}}', "answer": "", "reasoning": ""}
+            return {"error": None, "answer": "ok", "reasoning": ""}
+        orig = self.tune._stream_request
+        self.tune._stream_request = server
+        self.addCleanup(setattr, self.tune, "_stream_request", orig)
+        body = {"model": "new", "messages": [], "ignore_eos": True}
+        res = self.tune._request(self.eng, "http://x", body, "p", lambda *a, **k: None, 60)
+        self.assertIsNone(res["error"])
+        self.assertEqual(["ignore_eos" in b for b in sent], [True, False])   # retried without it
+        self.tune._request(self.eng, "http://x", body, "p", lambda *a, **k: None, 60)
+        self.assertEqual(len(sent), 3)   # later requests leave it out at once
+        self.assertNotIn("ignore_eos", sent[-1])
+        self.assertIn(("http://x", "new"), self.tune.no_ignore_eos(self.eng))
+
+    def test_other_request_errors_are_not_retried(self):
+        sent = []
+        orig = self.tune._stream_request
+        self.tune._stream_request = lambda eng, url, body, *a: sent.append(body) or {
+            "error": "server returned HTTP 400: bad messages", "answer": "", "reasoning": ""}
+        self.addCleanup(setattr, self.tune, "_stream_request", orig)
+        res = self.tune._request(self.eng, "http://x", {"model": "new", "ignore_eos": True}, "p",
+                                 lambda *a, **k: None, 60)
+        self.assertEqual((len(sent), res["error"]), (1, "server returned HTTP 400: bad messages"))
+
     def test_family_ignores_the_mtp_layer(self):
         # one GGUF lists KV heads per layer, none on the MTP (last) layer; the other gives one number
         d = self.tmp.name