tuieval 0.2.0.dev90__tar.gz → 0.2.0.dev92__tar.gz

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- A server that stops sending tokens mid-answer for 5 minutes (`stall_timeout_ms`) is a server error: retried, never counted against the model. Keep-alive pings no longer stretch a request past `request_timeout_ms`.
+- The run screen's queue shows what each running model is doing: waiting, thinking or answering with ~tokens so far, and how long it's been quiet, so a model you aren't streaming doesn't look stuck.
 - Results, Per question: the side-by-side summary shows any number of models (it stopped at 8).
 - `max_tokens` in pack.toml sets one pack's output budget; changing it reruns only that pack. Models whose context or endpoint can't fit it are skipped for that pack.
 - The run screen's queue keeps the running model in view, unless you scroll it yourself (scrolling back to it follows again).

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev90
+Version: 0.2.0.dev92
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/docs/models.md

@@ -191,6 +191,7 @@ A workspace's `models.toml` only needs what differs from these. `[defaults]` fil
 [defaults]
 repeat = 3
 request_timeout_ms = 1800000   # per request; long thinking runs need it
+stall_timeout_ms = 300000      # no tokens this long once answering began: the server stalled (retried, not counted)
 ready_timeout_s = 1200         # how long a model may take to load before giving up
 model_dirs = ["~/models"]      # where "Scan for models" looks for new GGUFs
 

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev90'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev90')
+__version__ = version = '0.2.0.dev92'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev92')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/src/tuieval/builtin.toml

@@ -5,6 +5,7 @@
 [defaults]
 repeat = 3
 request_timeout_ms = 1800000   # per request; long thinking runs need it
+stall_timeout_ms = 300000      # no tokens this long once answering began: the server stalled (retried, not counted)
 ready_timeout_s = 1200         # how long a model may take to load before giving up
 model_dirs = ["~/models"]      # where "Scan for models" looks for new GGUFs
 

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/src/tuieval/client.py

@@ -99,10 +99,17 @@ def count_tokens(base_url, text):
         return None
 
 
-def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, headers=None):
+def _stalled(stall_s):
+    return f"server error: stalled, no tokens for {stall_s:.0f} s after it started answering"
+
+
+def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, headers=None, stall_s=None):
     """POST {base_url}/v1/chat/completions with streaming. Returns a dict (see module docstring).
 
     on_delta(kind, text) is called for "reasoning" and "answer" text as it arrives.
+    timeout bounds the whole request, keep-alive comments (OpenRouter's ": OPENROUTER PROCESSING")
+    included; running out counts against the model, like any slow answer. stall_s: once tokens have
+    started, this long without one is the server stalling (a server error, never the model's answer).
     """
     stream = stream or Stream()
     body = dict(body, stream=True, stream_options={"include_usage": True})
@@ -111,7 +118,7 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
     out = {"answer": "", "reasoning": "", "tool_calls": [], "finish": None, "usage": {}, "timings": {},
            "ttft_s": None, "total_s": None, "error": None}
     reasoning, answer, tools = [], [], {}
-    start = time.time()
+    start = last = time.time()
     try:
         conn.request("POST", prefix + "/v1/chat/completions", body=json.dumps(body),
                      headers={"Content-Type": "application/json", "Authorization": "Bearer none", **(headers or {})})
@@ -123,6 +130,13 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
             out["error"] = f"server returned HTTP {r.status}: {r.read()[:300].decode('utf-8', 'replace')}"
             return out
         for raw in r:
+            now = time.time()
+            if now - start > timeout:
+                out["error"] = f"connection error: timed out after {timeout:.0f} s in all"
+                break
+            if stall_s and out["ttft_s"] is not None and now - last > stall_s:
+                out["error"] = _stalled(stall_s)
+                break
             line = raw.decode("utf-8", "replace").strip()
             if not line.startswith("data:"):
                 continue
@@ -147,8 +161,12 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
                     fn = tc.get("function") or {}
                     slot["name"] += fn.get("name") or ""
                     slot["arguments"] += fn.get("arguments") or ""
-                if (think or text or delta.get("tool_calls")) and out["ttft_s"] is None:
-                    out["ttft_s"] = time.time() - start
+                if think or text or delta.get("tool_calls"):
+                    last = time.time()
+                    if out["ttft_s"] is None:
+                        out["ttft_s"] = last - start
+                        if stall_s:      # a server that goes silent mid-answer: don't wait out the whole timeout
+                            stream.sock.settimeout(min(stall_s, timeout))
                 if think:
                     reasoning.append(think)
                     on_delta and on_delta("reasoning", think)
@@ -158,7 +176,8 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
     except (OSError, http.client.HTTPException, ValueError) as e:
         if stream.cancelled:
             raise Cancelled() from None
-        out["error"] = f"connection error: {e!r}"
+        stalled = stall_s and out["ttft_s"] is not None and isinstance(e, TimeoutError)
+        out["error"] = _stalled(stall_s) if stalled else f"connection error: {e!r}"
     finally:
         conn.close()
         out["total_s"] = time.time() - start

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/src/tuieval/engine.py

@@ -2229,6 +2229,7 @@ class Engine:
         self.emit("job_started", job=job, resumed=len(done_keys),
                   earlier_s=sum(s.get("wall_s") or 0 for s in job.earlier), sittings=len(job.earlier))
         timeout = self.cfg["defaults"]["request_timeout_ms"] / 1000
+        stall_s = (self.cfg["defaults"].get("stall_timeout_ms") or 0) / 1000 or None
         headers = self.request_headers(job.model)
         # a hosted API has no /tokenize to split reasoning from answer tokens (it reports the split)
         tokenize_url = None if self.cfg["servers"][job.model["server"]].get("api_key_env") else base_url
@@ -2253,7 +2254,8 @@ class Engine:
                         body = self._body(job, test, rep)
                         res = client.stream_chat(base_url, body,
                                                  lambda kind, text: self.emit("delta", stream=kind, text=text, label=job.label),
-                                                 timeout=timeout, stream=self._stream, headers=headers)
+                                                 timeout=timeout, stream=self._stream, headers=headers,
+                                                 stall_s=stall_s)
                     except client.Cancelled:
                         self._check()
                         raise Cancelled()

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/src/tuieval/tui.py

@@ -2104,6 +2104,7 @@ class RunScreen(Screen):
         self.follow = True
         self.buf = {"reasoning": [], "answer": []}
         self.buf_lock = threading.Lock()
+        self.req_live = {}                    # model label -> its request now: phase, ~tokens, started, last token
         self.started = time.time()
         self.req_started = None
         self.req_tokens = 0
@@ -2235,7 +2236,20 @@ class RunScreen(Screen):
             return  # the app is shutting down; the UI thread may be waiting on the engine
         if kind in ("request_started", "model_loading") and self.watch not in self.active():
             self.watch = d["job"].label if "job" in d else d["label"]   # follow the next model streaming
+        if kind in ("request_started", "server_retry"):   # here, not in handle(): deltas follow at once
+            with self.buf_lock:
+                self.req_live[d["job"].label if "job" in d else d["label"]] = {
+                    "phase": "waiting", "tokens": 0, "started": time.time(), "last": None}
+        elif kind == "request_done":
+            with self.buf_lock:
+                self.req_live.pop(d["job"].label, None)
         if kind == "delta":  # very frequent: buffer, flushed every 100 ms on the UI thread
+            with self.buf_lock:
+                lv = self.req_live.get(d.get("label"))
+                if lv:
+                    lv["phase"] = "thinking" if d["stream"] == "reasoning" else "answering"
+                    lv["tokens"] += max(1, len(d["text"]) // 4)
+                    lv["last"] = time.time()
             if d.get("label", self.watch) != self.watch:
                 return       # another model running alongside: v switches to it
             with self.buf_lock:
@@ -2439,7 +2453,8 @@ class RunScreen(Screen):
         elapsed = (j.finished or time.time()) - j.started if j.started else 0
         elapsed += sum(s.get("wall_s") or 0 for s in j.earlier)   # a resumed pack: its earlier sittings too
         # update_width: columns start at their header's width; "12" under ✓ would otherwise show as "1"
-        for col, value in (("status", f"[{style}]{j.status}[/{style}]"),
+        status = f"[{style}]{j.status}[/{style}]" + (self.live_note(j.label) if j.status == "running" else "")
+        for col, value in (("status", status),
                            ("progress", f"{j.done}/{j.total}" if j.status != "skipped" or j.done else ""),
                            ("pass", f"[green]{j.passed}[/green]" if j.passed else ""),
                            ("fail", f"[red]{j.failed}[/red]" if j.failed else ""),
@@ -2447,6 +2462,21 @@ class RunScreen(Screen):
                            ("note", short_note(j.note))):
             q.update_cell(j.key, col, value, update_width=True)
 
+    def live_note(self, label):
+        """What a running model's request is doing now, so a slow model doesn't look stuck: waiting for
+        its first token, or thinking/answering with ~tokens so far, and how long it's been quiet."""
+        with self.buf_lock:
+            lv = dict(self.req_live.get(label) or {})
+        if not lv:
+            return ""
+        now = time.time()
+        if lv["last"] is None:
+            return f" [dim]· waiting {fmt_secs(now - lv['started'])}[/dim]"
+        note = f" [dim]· {lv['phase']} ~{lv['tokens']:,} tok[/dim]"
+        quiet = now - lv["last"]
+        return note + (f" [yellow]· quiet {fmt_secs(quiet)}[/yellow]" if quiet >= self.QUIET_S else "")
+
+    QUIET_S = 30       # a running model's row says how long it's been without a token after this long
     LIVE_ANSWERS = 5   # a pack's own times this session take over from its history after this many
 
     def eta_seconds(self):

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/tests/mock_server.py

@@ -9,6 +9,8 @@ fixed   always "ANSWER: 42"
 unavailable  HTTP 404 "No endpoints found" for every chat request
 
 Streams like llama.cpp: reasoning_content then content deltas, usage and timings at the end.
+--stall S: after the first token, S seconds of nothing but keep-alive comments (": OPENROUTER
+PROCESSING"), or of silence with --silent; --stall-before S: the same before the first token.
 An answer longer than the request's max_tokens (one token per word) is cut off with finish
 "length". --max-output N also serves an OpenRouter-style endpoints list with that output limit.
 """
@@ -50,6 +52,7 @@ def reply_for(test, mode):
 
 class Handler(BaseHTTPRequestHandler):
     answers, mode, model, delay, max_output = {}, "oracle", "mock", 0.0, None
+    stall, stall_before, silent = 0.0, 0.0, False
 
     def log_message(self, *a):
         pass
@@ -98,7 +101,17 @@ class Handler(BaseHTTPRequestHandler):
             self.wfile.write(f"data: {json.dumps(obj)}\n\n".encode())
             self.wfile.flush()
 
+        def wait(secs):
+            end = time.time() + secs
+            while time.time() < end:
+                if not self.silent:
+                    self.wfile.write(b": OPENROUTER PROCESSING\n\n")
+                    self.wfile.flush()
+                time.sleep(0.2)
+
+        wait(self.stall_before)
         send({"choices": [{"delta": {"reasoning_content": "Thinking it over. "}}]})
+        wait(self.stall)
         time.sleep(self.delay or 0.005)
         for i in range(0, len(text), 12):
             send({"choices": [{"delta": {"content": text[i:i + 12]}}]})
@@ -128,9 +141,13 @@ def main(argv=None):
     p.add_argument("--model", default="mock")
     p.add_argument("--delay", type=float, default=0.0, help="seconds each answer takes")
     p.add_argument("--max-output", type=int)
+    p.add_argument("--stall", type=float, default=0.0)
+    p.add_argument("--stall-before", type=float, default=0.0)
+    p.add_argument("--silent", action="store_true")
     a, _ = p.parse_known_args(argv)   # other flags (e.g. speed knobs under test) are accepted and ignored
     Handler.answers, Handler.mode, Handler.model, Handler.delay = load_answers(a.packs), a.mode, a.model, a.delay
     Handler.max_output = a.max_output
+    Handler.stall, Handler.stall_before, Handler.silent = a.stall, a.stall_before, a.silent
     srv = Server(("127.0.0.1", a.port), Handler)
     print(f"mock server on 127.0.0.1:{a.port} ({a.mode}, {len(Handler.answers)} known questions)", flush=True)
     try:

{tuieval-0.2.0.dev90 → tuieval-0.2.0.dev92}/tests/test_tuieval.py

@@ -2373,5 +2373,77 @@ class Logs(unittest.TestCase):
             self.assertEqual(out, "── chat ──\nchat line\n── coder-q4 ──\nq4 server started\n")
 
 
+class StallGuard(unittest.TestCase):
+    """A server that stops sending tokens mid-answer (keep-alive comments or silence) is a server error,
+    not the model's answer; keep-alives can't stretch a request past its overall timeout."""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp = tempfile.TemporaryDirectory()
+        os.makedirs(os.path.join(cls.tmp.name, "packs"))
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.tmp.cleanup()
+
+    def ask(self, *flags, **kw):
+        from tuieval import client
+        mock = Mock(os.path.join(self.tmp.name, "packs"), "fixed", None, 0.0, *flags)
+        try:
+            t = time.time()
+            res = client.stream_chat(f"http://127.0.0.1:{mock.port}/v1", {"model": "mock", "messages": [
+                {"role": "user", "content": "q"}]}, **kw)
+            return res, time.time() - t
+        finally:
+            mock.stop()
+
+    def test_keepalives_after_first_token_stall(self):
+        from tuieval import client
+        res, took = self.ask("--stall", "5", timeout=30, stall_s=1)
+        self.assertTrue(res["error"].startswith("server error: stalled"), res["error"])
+        self.assertTrue(client.is_server_error(res["error"]))
+        self.assertLess(took, 4)
+
+    def test_silence_after_first_token_stalls(self):
+        from tuieval import client
+        res, took = self.ask("--stall", "5", "--silent", timeout=30, stall_s=1)
+        self.assertTrue(client.is_server_error(res["error"]), res["error"])
+        self.assertIn("stalled", res["error"])
+        self.assertLess(took, 4)
+
+    def test_long_wait_before_first_token_is_not_a_stall(self):
+        res, _ = self.ask("--stall-before", "1.5", timeout=30, stall_s=1)
+        self.assertIsNone(res["error"])
+        self.assertIn("42", res["answer"])
+
+    def test_keepalives_dont_outlast_the_timeout(self):
+        from tuieval import client
+        res, took = self.ask("--stall-before", "5", timeout=1, stall_s=1)
+        self.assertIn("timed out", res["error"])
+        self.assertFalse(client.is_server_error(res["error"]))   # slow, like any timeout: counts
+        self.assertLess(took, 4)
+
+    def test_run_screen_shows_each_running_models_progress(self):
+        from types import SimpleNamespace
+        from tuieval import tui
+        calls = []
+        fake = SimpleNamespace(app=SimpleNamespace(closing=False, call_from_thread=lambda *a: calls.append(a)),
+                               watch="a", active=lambda: ["a", "b"], buf={"reasoning": [], "answer": []},
+                               buf_lock=threading.Lock(), req_live={}, QUIET_S=30, handle=None)
+        fake.live_note = lambda label: tui.RunScreen.live_note(fake, label)
+        ev = lambda kind, **d: tui.RunScreen.on_engine_event(fake, kind, **d)
+        ev("request_started", job=SimpleNamespace(label="b"), test="t")
+        self.assertIn("waiting", fake.live_note("b"))
+        ev("delta", stream="reasoning", text="x" * 400, label="b")      # b isn't streamed: still counted
+        self.assertEqual(fake.buf["reasoning"], [])
+        self.assertIn("thinking ~100 tok", fake.live_note("b"))
+        ev("delta", stream="answer", text="y" * 40, label="b")
+        self.assertIn("answering ~110 tok", fake.live_note("b"))
+        fake.req_live["b"]["last"] -= 90
+        self.assertIn("quiet 1m30s", fake.live_note("b"))
+        ev("request_done", job=SimpleNamespace(label="b"), record={})
+        self.assertEqual(fake.live_note("b"), "")
+
+
 if __name__ == "__main__":
     unittest.main()