tuieval 0.2.0.dev125__tar.gz → 0.2.0.dev127__tar.gz

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev127}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix: a request that runs out of time says `time limit: timed out after N s`, not `connection error`, and `tuieval compare` counts it against the model like the run does (it was left out as an error).
 - Agent packs (`kind = "agent"`): each test is a small project with a task, worked on by a command-line agent you define in `[agents]`, using the model tuieval serves, and graded by the task's visible and hidden tests. Results show the agent's stats, diff and transcript; `tuieval compare` adds an Agent tasks table; `tuieval selftest` checks each task's reference and wrong fixes. See docs/agent-packs.md.
 - Answers that used more tokens than the `max_tokens` sent (a provider that didn't stop them) are marked over budget: `(n over budget)` in the PTA index, an **over budget** column, `over` in Per question and a note in the answer view. `[defaults] over_budget = "fail"` fails them instead. Works on stored results.
 - Fix: Results → Per question opened the answers of the model one column to the right of the cell picked, and crashed on the last model's column.

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev127}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev125
+Version: 0.2.0.dev127
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev127}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev125'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev125')
+__version__ = version = '0.2.0.dev127'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev127')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev127}/src/tuieval/client.py

@@ -153,7 +153,7 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
         for raw in r:
             now = time.time()
             if now - start > timeout:
-                out["error"] = f"connection error: timed out after {timeout:.0f} s in all"
+                out["error"] = f"time limit: timed out after {timeout:.0f} s in all"
                 break
             if stall_s and out["ttft_s"] is not None and now - last > stall_s:
                 out["error"] = _stalled(stall_s)

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev127}/src/tuieval/compare.py

@@ -49,7 +49,8 @@ def _load_native(data):
             "model": m["label"], "suite": pack["name"], "test": f"{pack['name']}: {r['test']}",
             "name": f"{pack['name']}: {r['description']}",
             "ok": bool(r["pass"]), "score": float(r["score"]), "reason": r["reason"],
-            "error": r["reason"].startswith(("server returned", "connection error")),
+            # a timeout is the model too slow (client.is_server_error), not an error; older results called it one
+            "error": r["reason"].startswith(("server returned", "connection error")) and "timed out" not in r["reason"],
             "truncated": r.get("finish") == "length", "output": r.get("answer") or "",
             # tokens past the max_tokens sent, which the provider didn't stop (client.mark_over_budget)
             "over_budget": r.get("over_budget"),

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev127}/tests/mock_server.py

@@ -56,6 +56,7 @@ class Handler(BaseHTTPRequestHandler):
     answers, mode, model, delay, max_output = {}, "oracle", "mock", 0.0, None
     stall, stall_before, silent = 0.0, 0.0, False
     die_after = None
+    hold, answered = None, 0   # while the hold file exists, answers after the first wait (a run that stays going)
 
     def log_message(self, *a):
         pass
@@ -120,6 +121,8 @@ class Handler(BaseHTTPRequestHandler):
                     self.wfile.flush()
                 time.sleep(0.2)
 
+        while self.hold and Handler.answered and os.path.exists(self.hold):
+            wait(0.2)
         wait(self.stall_before)
         send({"choices": [{"delta": {"reasoning_content": "Thinking it over. "}}]})
         wait(self.stall)
@@ -136,6 +139,7 @@ class Handler(BaseHTTPRequestHandler):
                           "prompt_per_second": 500.0, "cache_n": 0}})
         die_if_due()
         self.wfile.write(b"data: [DONE]\n\n")
+        Handler.answered += 1
 
 
 class Server(ThreadingHTTPServer):
@@ -157,11 +161,12 @@ def main(argv=None):
     p.add_argument("--stall-before", type=float, default=0.0)
     p.add_argument("--silent", action="store_true")
     p.add_argument("--die-after", type=int)
+    p.add_argument("--hold", help="while this file exists, every answer after the first waits")
     a, _ = p.parse_known_args(argv)   # other flags (e.g. speed knobs under test) are accepted and ignored
     Handler.answers, Handler.mode, Handler.model, Handler.delay = load_answers(a.packs), a.mode, a.model, a.delay
     Handler.max_output = a.max_output
     Handler.stall, Handler.stall_before, Handler.silent = a.stall, a.stall_before, a.silent
-    Handler.die_after = a.die_after
+    Handler.die_after, Handler.hold = a.die_after, a.hold
     srv = Server(("127.0.0.1", a.port), Handler)
     print(f"mock server on 127.0.0.1:{a.port} ({a.mode}, {len(Handler.answers)} known questions)", flush=True)
     try:

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev127}/tests/test_tuieval.py

@@ -1412,8 +1412,12 @@ class AddToRun(unittest.TestCase):
 
     def test_tui_never_queues_the_same_thing_twice(self):
         """Start while a run is going never adds or queues a model and pack already running or queued."""
+        hold = os.path.join(self.tmp.name, "hold")   # the first run stays going until every pick is made
+        write(hold, "")
+        path = os.path.join(self.ws, "models.toml")
+        write(path, read(path).replace('"--delay", "0.2"', f'"--delay", "0.2", "--hold", {json.dumps(hold)}'))
         code = textwrap.dedent("""
-            import asyncio, json, time
+            import asyncio, json, os, time
             from tuieval.tui import EvalsApp, RunScreen, SetupScreen, ChoiceScreen
             from textual.widgets import SelectionList
             async def pick(app, pilot, models, packs=("apps",)):
@@ -1459,6 +1463,7 @@ class AddToRun(unittest.TestCase):
                     if isinstance(s, ChoiceScreen):
                         s.dismiss("queue")
                     await pilot.pause()
+                    os.remove(os.environ["HOLD"])
                     deadline = time.time() + 300   # the queued runs start one after another: wait for all three
                     while time.time() < deadline:
                         await pilot.pause(0.2)
@@ -1469,7 +1474,7 @@ class AddToRun(unittest.TestCase):
             asyncio.run(go())
         """)
         p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=600,
-                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""))
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS="", HOLD=hold))
         out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
         self.assertIsInstance(out, dict, out)
         self.assertEqual(out["same"], [False, 0])
@@ -3005,7 +3010,7 @@ class StallGuard(unittest.TestCase):
     def test_keepalives_dont_outlast_the_timeout(self):
         from tuieval import client
         res, took = self.ask("--stall-before", "5", timeout=1, stall_s=1)
-        self.assertIn("timed out", res["error"])
+        self.assertTrue(res["error"].startswith("time limit: timed out"), res["error"])
         self.assertFalse(client.is_server_error(res["error"]))   # slow, like any timeout: counts
         self.assertLess(took, 4)