tuieval 0.2.0.dev125__tar.gz → 0.2.0.dev126__tar.gz

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev126}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix: a request that runs out of time says `time limit: timed out after N s`, not `connection error`, and `tuieval compare` counts it against the model like the run does (it was left out as an error).
 - Agent packs (`kind = "agent"`): each test is a small project with a task, worked on by a command-line agent you define in `[agents]`, using the model tuieval serves, and graded by the task's visible and hidden tests. Results show the agent's stats, diff and transcript; `tuieval compare` adds an Agent tasks table; `tuieval selftest` checks each task's reference and wrong fixes. See docs/agent-packs.md.
 - Answers that used more tokens than the `max_tokens` sent (a provider that didn't stop them) are marked over budget: `(n over budget)` in the PTA index, an **over budget** column, `over` in Per question and a note in the answer view. `[defaults] over_budget = "fail"` fails them instead. Works on stored results.
 - Fix: Results → Per question opened the answers of the model one column to the right of the cell picked, and crashed on the last model's column.

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev126}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev125
+Version: 0.2.0.dev126
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev126}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev125'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev125')
+__version__ = version = '0.2.0.dev126'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev126')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev126}/src/tuieval/client.py

@@ -153,7 +153,7 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
         for raw in r:
             now = time.time()
             if now - start > timeout:
-                out["error"] = f"connection error: timed out after {timeout:.0f} s in all"
+                out["error"] = f"time limit: timed out after {timeout:.0f} s in all"
                 break
             if stall_s and out["ttft_s"] is not None and now - last > stall_s:
                 out["error"] = _stalled(stall_s)

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev126}/src/tuieval/compare.py

@@ -49,7 +49,8 @@ def _load_native(data):
             "model": m["label"], "suite": pack["name"], "test": f"{pack['name']}: {r['test']}",
             "name": f"{pack['name']}: {r['description']}",
             "ok": bool(r["pass"]), "score": float(r["score"]), "reason": r["reason"],
-            "error": r["reason"].startswith(("server returned", "connection error")),
+            # a timeout is the model too slow (client.is_server_error), not an error; older results called it one
+            "error": r["reason"].startswith(("server returned", "connection error")) and "timed out" not in r["reason"],
             "truncated": r.get("finish") == "length", "output": r.get("answer") or "",
             # tokens past the max_tokens sent, which the provider didn't stop (client.mark_over_budget)
             "over_budget": r.get("over_budget"),

{tuieval-0.2.0.dev125 → tuieval-0.2.0.dev126}/tests/test_tuieval.py

@@ -3005,7 +3005,7 @@ class StallGuard(unittest.TestCase):
     def test_keepalives_dont_outlast_the_timeout(self):
         from tuieval import client
         res, took = self.ask("--stall-before", "5", timeout=1, stall_s=1)
-        self.assertIn("timed out", res["error"])
+        self.assertTrue(res["error"].startswith("time limit: timed out"), res["error"])
         self.assertFalse(client.is_server_error(res["error"]))   # slow, like any timeout: counts
         self.assertLess(took, 4)