tuieval 0.2.0.dev89__py3-none-any.whl → 0.2.0.dev92__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev89'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev89')
+__version__ = version = '0.2.0.dev92'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev92')
 
 __commit_id__ = commit_id = None

tuieval/builtin.toml

@@ -5,6 +5,7 @@
 [defaults]
 repeat = 3
 request_timeout_ms = 1800000   # per request; long thinking runs need it
+stall_timeout_ms = 300000      # no tokens this long once answering began: the server stalled (retried, not counted)
 ready_timeout_s = 1200         # how long a model may take to load before giving up
 model_dirs = ["~/models"]      # where "Scan for models" looks for new GGUFs
 

tuieval/client.py

@@ -99,10 +99,17 @@ def count_tokens(base_url, text):
         return None
 
 
-def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, headers=None):
+def _stalled(stall_s):
+    return f"server error: stalled, no tokens for {stall_s:.0f} s after it started answering"
+
+
+def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, headers=None, stall_s=None):
     """POST {base_url}/v1/chat/completions with streaming. Returns a dict (see module docstring).
 
     on_delta(kind, text) is called for "reasoning" and "answer" text as it arrives.
+    timeout bounds the whole request, keep-alive comments (OpenRouter's ": OPENROUTER PROCESSING")
+    included; running out counts against the model, like any slow answer. stall_s: once tokens have
+    started, this long without one is the server stalling (a server error, never the model's answer).
     """
     stream = stream or Stream()
     body = dict(body, stream=True, stream_options={"include_usage": True})
@@ -111,7 +118,7 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
     out = {"answer": "", "reasoning": "", "tool_calls": [], "finish": None, "usage": {}, "timings": {},
            "ttft_s": None, "total_s": None, "error": None}
     reasoning, answer, tools = [], [], {}
-    start = time.time()
+    start = last = time.time()
     try:
         conn.request("POST", prefix + "/v1/chat/completions", body=json.dumps(body),
                      headers={"Content-Type": "application/json", "Authorization": "Bearer none", **(headers or {})})
@@ -123,6 +130,13 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
             out["error"] = f"server returned HTTP {r.status}: {r.read()[:300].decode('utf-8', 'replace')}"
             return out
         for raw in r:
+            now = time.time()
+            if now - start > timeout:
+                out["error"] = f"connection error: timed out after {timeout:.0f} s in all"
+                break
+            if stall_s and out["ttft_s"] is not None and now - last > stall_s:
+                out["error"] = _stalled(stall_s)
+                break
             line = raw.decode("utf-8", "replace").strip()
             if not line.startswith("data:"):
                 continue
@@ -147,8 +161,12 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
                     fn = tc.get("function") or {}
                     slot["name"] += fn.get("name") or ""
                     slot["arguments"] += fn.get("arguments") or ""
-                if (think or text or delta.get("tool_calls")) and out["ttft_s"] is None:
-                    out["ttft_s"] = time.time() - start
+                if think or text or delta.get("tool_calls"):
+                    last = time.time()
+                    if out["ttft_s"] is None:
+                        out["ttft_s"] = last - start
+                        if stall_s:      # a server that goes silent mid-answer: don't wait out the whole timeout
+                            stream.sock.settimeout(min(stall_s, timeout))
                 if think:
                     reasoning.append(think)
                     on_delta and on_delta("reasoning", think)
@@ -158,7 +176,8 @@ def stream_chat(base_url, body, on_delta=None, timeout=1800, stream=None, header
     except (OSError, http.client.HTTPException, ValueError) as e:
         if stream.cancelled:
             raise Cancelled() from None
-        out["error"] = f"connection error: {e!r}"
+        stalled = stall_s and out["ttft_s"] is not None and isinstance(e, TimeoutError)
+        out["error"] = _stalled(stall_s) if stalled else f"connection error: {e!r}"
     finally:
         conn.close()
         out["total_s"] = time.time() - start

tuieval/engine.py

@@ -2229,6 +2229,7 @@ class Engine:
         self.emit("job_started", job=job, resumed=len(done_keys),
                   earlier_s=sum(s.get("wall_s") or 0 for s in job.earlier), sittings=len(job.earlier))
         timeout = self.cfg["defaults"]["request_timeout_ms"] / 1000
+        stall_s = (self.cfg["defaults"].get("stall_timeout_ms") or 0) / 1000 or None
         headers = self.request_headers(job.model)
         # a hosted API has no /tokenize to split reasoning from answer tokens (it reports the split)
         tokenize_url = None if self.cfg["servers"][job.model["server"]].get("api_key_env") else base_url
@@ -2253,7 +2254,8 @@ class Engine:
                         body = self._body(job, test, rep)
                         res = client.stream_chat(base_url, body,
                                                  lambda kind, text: self.emit("delta", stream=kind, text=text, label=job.label),
-                                                 timeout=timeout, stream=self._stream, headers=headers)
+                                                 timeout=timeout, stream=self._stream, headers=headers,
+                                                 stall_s=stall_s)
                     except client.Cancelled:
                         self._check()
                         raise Cancelled()

tuieval/tui.py

@@ -2104,6 +2104,7 @@ class RunScreen(Screen):
         self.follow = True
         self.buf = {"reasoning": [], "answer": []}
         self.buf_lock = threading.Lock()
+        self.req_live = {}                    # model label -> its request now: phase, ~tokens, started, last token
         self.started = time.time()
         self.req_started = None
         self.req_tokens = 0
@@ -2235,7 +2236,20 @@ class RunScreen(Screen):
             return  # the app is shutting down; the UI thread may be waiting on the engine
         if kind in ("request_started", "model_loading") and self.watch not in self.active():
             self.watch = d["job"].label if "job" in d else d["label"]   # follow the next model streaming
+        if kind in ("request_started", "server_retry"):   # here, not in handle(): deltas follow at once
+            with self.buf_lock:
+                self.req_live[d["job"].label if "job" in d else d["label"]] = {
+                    "phase": "waiting", "tokens": 0, "started": time.time(), "last": None}
+        elif kind == "request_done":
+            with self.buf_lock:
+                self.req_live.pop(d["job"].label, None)
         if kind == "delta":  # very frequent: buffer, flushed every 100 ms on the UI thread
+            with self.buf_lock:
+                lv = self.req_live.get(d.get("label"))
+                if lv:
+                    lv["phase"] = "thinking" if d["stream"] == "reasoning" else "answering"
+                    lv["tokens"] += max(1, len(d["text"]) // 4)
+                    lv["last"] = time.time()
             if d.get("label", self.watch) != self.watch:
                 return       # another model running alongside: v switches to it
             with self.buf_lock:
@@ -2439,7 +2453,8 @@ class RunScreen(Screen):
         elapsed = (j.finished or time.time()) - j.started if j.started else 0
         elapsed += sum(s.get("wall_s") or 0 for s in j.earlier)   # a resumed pack: its earlier sittings too
         # update_width: columns start at their header's width; "12" under ✓ would otherwise show as "1"
-        for col, value in (("status", f"[{style}]{j.status}[/{style}]"),
+        status = f"[{style}]{j.status}[/{style}]" + (self.live_note(j.label) if j.status == "running" else "")
+        for col, value in (("status", status),
                            ("progress", f"{j.done}/{j.total}" if j.status != "skipped" or j.done else ""),
                            ("pass", f"[green]{j.passed}[/green]" if j.passed else ""),
                            ("fail", f"[red]{j.failed}[/red]" if j.failed else ""),
@@ -2447,6 +2462,21 @@ class RunScreen(Screen):
                            ("note", short_note(j.note))):
             q.update_cell(j.key, col, value, update_width=True)
 
+    def live_note(self, label):
+        """What a running model's request is doing now, so a slow model doesn't look stuck: waiting for
+        its first token, or thinking/answering with ~tokens so far, and how long it's been quiet."""
+        with self.buf_lock:
+            lv = dict(self.req_live.get(label) or {})
+        if not lv:
+            return ""
+        now = time.time()
+        if lv["last"] is None:
+            return f" [dim]· waiting {fmt_secs(now - lv['started'])}[/dim]"
+        note = f" [dim]· {lv['phase']} ~{lv['tokens']:,} tok[/dim]"
+        quiet = now - lv["last"]
+        return note + (f" [yellow]· quiet {fmt_secs(quiet)}[/yellow]" if quiet >= self.QUIET_S else "")
+
+    QUIET_S = 30       # a running model's row says how long it's been without a token after this long
     LIVE_ANSWERS = 5   # a pack's own times this session take over from its history after this many
 
     def eta_seconds(self):
@@ -2983,10 +3013,9 @@ class ResultsScreen(Screen):
         table.clear(columns=True)
         title = self.query_one("#pq-compare-title", Static)
         shared = [t for t in tests if all(m in t["cells"] for m in models)]
-        if len(models) < 2 or len(models) > 8:
+        if len(models) < 2:
             table.display = False
-            title.update("[dim]Pick 2 to 8 models (Models button) for a side-by-side summary.[/dim]"
-                         if len(models) > 8 else "")
+            title.update("")
             return
         table.display = True
         if not shared:

{tuieval-0.2.0.dev89.dist-info → tuieval-0.2.0.dev92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev89
+Version: 0.2.0.dev92
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev89.dist-info → tuieval-0.2.0.dev92.dist-info}/RECORD

@@ -1,12 +1,12 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=XiUzkOIwjrCyySM-wvAr3iW2t63wQjeKE_BTt6XJQng,535
-tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
+tuieval/_version.py,sha256=LTyNMvKP3YQzCLuW_fhXY3Ipvz3I7zhbkkf5Q0Fzxqs,535
+tuieval/builtin.toml,sha256=Nwu7O0qFzcrDAxXO_VZtDPvjVwxZ29OtYOW0BPihZGs,4410
 tuieval/cli.py,sha256=VwiaU1F9bZtU8C-gEowJq58kPeYQY4t6SY_HwrV-qgc,4702
-tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
+tuieval/client.py,sha256=wtACBVsngtWopWWhIc4bB6mEUEi31dvvDePN-PjmXbM,11122
 tuieval/compare.py,sha256=uXU_Yzf4zDSHoupHQrTmBhsfc8wCyI1nMaZQN81ybiA,30522
 tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
-tuieval/engine.py,sha256=BS2BXLaxtw3RFI6-5PDvhV5VLh5v0m0cB_F3fPC5MtM,125105
+tuieval/engine.py,sha256=96GWzKzMQ_KaG6wXrIuBiKWFCcblXWL9cQs89QYgj7Y,125256
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
 tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
@@ -18,7 +18,7 @@ tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=FU3bvu3jt9ZWg3YLsXNINbdeClXMioyc4zQ4yEhmBQs,47855
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=yD-3pQJ7I71r8lg49m0aWD5SlfA_Bk9BIz6XUJj8_UM,177016
+tuieval/tui.py,sha256=a3ZANtRLq9bBSIdxRj2Ksgj8S5lF8shSMNBxpIzH6rM,178622
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -47,8 +47,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev89.dist-info/METADATA,sha256=t3hBuHdUYssk-1LBJ7YfWAHkTPjsTXR9CnErwPjDKo8,14580
-tuieval-0.2.0.dev89.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev89.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev89.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev89.dist-info/RECORD,,
+tuieval-0.2.0.dev92.dist-info/METADATA,sha256=1zUki84oRDfw_gL4VqsvwMsDMxV35smy8dZheMab-RI,14580
+tuieval-0.2.0.dev92.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev92.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev92.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev92.dist-info/RECORD,,