tuieval 0.2.0.dev42__py3-none-any.whl → 0.2.0.dev43__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev42'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev42')
+__version__ = version = '0.2.0.dev43'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev43')
 
 __commit_id__ = commit_id = None

tuieval/client.py

@@ -17,6 +17,10 @@ import time
 import urllib.parse
 
 SERVER_ERROR_PREFIXES = ("server returned HTTP 5", "server returned HTTP 429", "server error", "connection error")
+# the model can't be reached there at all (a bad key, no credits, or a hosted API with no provider serving it, e.g.
+# OpenRouter's 404 "No endpoints found"): not the model's answer either, and waiting won't fix it
+UNAVAILABLE_PREFIXES = ("server returned HTTP 401", "server returned HTTP 402", "server returned HTTP 403",
+                        "server returned HTTP 404")
 
 
 def is_server_error(error):
@@ -27,9 +31,15 @@ def is_server_error(error):
         and "timeout" not in error.lower()
 
 
+def is_unavailable(error):
+    """True for an error saying the model isn't available on that server at all: HTTP 401, 402, 403 or
+    404. Never the model's answer, and not worth retrying."""
+    return bool(error) and error.startswith(UNAVAILABLE_PREFIXES)
+
+
 def server_error_row(rec):
     """A stored answer that was really a server failure (never a judgement of the model)."""
-    return not rec.get("pass") and is_server_error(rec.get("reason"))
+    return not rec.get("pass") and (is_server_error(rec.get("reason")) or is_unavailable(rec.get("reason")))
 
 
 class Cancelled(Exception):

tuieval/engine.py

@@ -2105,6 +2105,9 @@ class Engine:
                     self._check()
                     if res["error"] and not alive():
                         raise ModelFailed(f"the server stopped: {res['error']}")
+                    if client.is_unavailable(res["error"]):   # never an answer; retrying won't help
+                        raise ModelFailed(f"the model isn't available there (not counted against it): "
+                                          f"{res['error'][:300]}")
                     if not client.is_server_error(res["error"]):
                         break
                     # the server failed, not the model: never recorded as an answer

{tuieval-0.2.0.dev42.dist-info → tuieval-0.2.0.dev43.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev42
+Version: 0.2.0.dev43
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev42.dist-info → tuieval-0.2.0.dev43.dist-info}/RECORD

@@ -1,12 +1,12 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=DfGXotYkwonAQhxctMA784jbVvrHBcDTr170a85efeI,535
+tuieval/_version.py,sha256=5PoXfeeUc_RJHZEcbrdVeWQpyK7qC0e8oom2QPT307Y,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=kI1P0a7TMS-daHjZjLDBdBDPOwBCmPw1qjw6VDlY258,4505
-tuieval/client.py,sha256=Vui7ucVv_GiKObGoVS_EWSZFUKPou9PWmutqMTrahXk,9313
+tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
 tuieval/compare.py,sha256=SHN_o_nlppFcHMBqBZRITIkeQDPaR_JxhUZItQLMBwo,26270
 tuieval/doctor.py,sha256=y1C_ggjd8oikXTrovgTBf26wjThP8EoVkfTVeO_n5dQ,6070
-tuieval/engine.py,sha256=xTDlcU2nPGcSvv6a45gLTsMU7p7ZvNxmzrkGPMe9vfQ,113950
+tuieval/engine.py,sha256=w3KsgxzFAMcRyVzFGQXhTgN3d9qYRtAZZxNLXZOgnrQ,114222
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/machines.py,sha256=ImEihNj8QlBRdCHLED4QIdjJUvjVtZgo6DiVU5h6Zzs,13415
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
@@ -46,8 +46,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev42.dist-info/METADATA,sha256=dGI2rKY-NRJBTTx7nc3LvsgWxkpZwaaGivddPyzBDrw,13881
-tuieval-0.2.0.dev42.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev42.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev42.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev42.dist-info/RECORD,,
+tuieval-0.2.0.dev43.dist-info/METADATA,sha256=P6oHoB_GI8wBLV-_DUqU4TggqZdTVGtz5glAfpIn2jU,13881
+tuieval-0.2.0.dev43.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev43.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev43.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev43.dist-info/RECORD,,