tuieval 0.2.0.dev42__tar.gz → 0.2.0.dev43__tar.gz

{tuieval-0.2.0.dev42 → tuieval-0.2.0.dev43}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- A model a server won't serve (HTTP 401-404, e.g. OpenRouter's "No endpoints found") is never judged on it: those answers don't count, and the model stops at once instead of failing every question.
 - PTA index: privacy, time (total, relative to the fastest) and accuracy per model, as a triangle in Results, `tuieval pta` and the report.
 - Run several models at a time: `parallel_models` per machine, **Models at a time** in the TUI, or `tuieval run --parallel N` (default 1). Models wait for memory to fit, and answers note what ran alongside them.
 - More evals during a run: `n` opens Setup, and Start adds them to the run going (same tier) or queues them after it.

{tuieval-0.2.0.dev42 → tuieval-0.2.0.dev43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev42
+Version: 0.2.0.dev43
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev42 → tuieval-0.2.0.dev43}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev42'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev42')
+__version__ = version = '0.2.0.dev43'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev43')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev42 → tuieval-0.2.0.dev43}/src/tuieval/client.py

@@ -17,6 +17,10 @@ import time
 import urllib.parse
 
 SERVER_ERROR_PREFIXES = ("server returned HTTP 5", "server returned HTTP 429", "server error", "connection error")
+# the model can't be reached there at all (a bad key, no credits, or a hosted API with no provider serving it, e.g.
+# OpenRouter's 404 "No endpoints found"): not the model's answer either, and waiting won't fix it
+UNAVAILABLE_PREFIXES = ("server returned HTTP 401", "server returned HTTP 402", "server returned HTTP 403",
+                        "server returned HTTP 404")
 
 
 def is_server_error(error):
@@ -27,9 +31,15 @@ def is_server_error(error):
         and "timeout" not in error.lower()
 
 
+def is_unavailable(error):
+    """True for an error saying the model isn't available on that server at all: HTTP 401, 402, 403 or
+    404. Never the model's answer, and not worth retrying."""
+    return bool(error) and error.startswith(UNAVAILABLE_PREFIXES)
+
+
 def server_error_row(rec):
     """A stored answer that was really a server failure (never a judgement of the model)."""
-    return not rec.get("pass") and is_server_error(rec.get("reason"))
+    return not rec.get("pass") and (is_server_error(rec.get("reason")) or is_unavailable(rec.get("reason")))
 
 
 class Cancelled(Exception):

{tuieval-0.2.0.dev42 → tuieval-0.2.0.dev43}/src/tuieval/engine.py

@@ -2105,6 +2105,9 @@ class Engine:
                     self._check()
                     if res["error"] and not alive():
                         raise ModelFailed(f"the server stopped: {res['error']}")
+                    if client.is_unavailable(res["error"]):   # never an answer; retrying won't help
+                        raise ModelFailed(f"the model isn't available there (not counted against it): "
+                                          f"{res['error'][:300]}")
                     if not client.is_server_error(res["error"]):
                         break
                     # the server failed, not the model: never recorded as an answer

{tuieval-0.2.0.dev42 → tuieval-0.2.0.dev43}/tests/mock_server.py

@@ -6,6 +6,7 @@ oracle  answers each question with its test's `reference` (or the expected tool
         correct grader passes it
 wrong   answers with the test's first `wrong` entry (or "ANSWER: 0")
 fixed   always "ANSWER: 42"
+unavailable  HTTP 404 "No endpoints found" for every chat request
 
 Streams like llama.cpp: reasoning_content then content deltas, usage and timings at the end.
 """
@@ -77,6 +78,8 @@ class Handler(BaseHTTPRequestHandler):
         if isinstance(content, list):
             content = " ".join(p.get("text", "") for p in content if isinstance(p, dict))
         test = self.answers.get(" ".join(str(content).split()))
+        if self.mode == "unavailable":   # like OpenRouter with no provider serving the model
+            return self._json({"error": {"message": f"No endpoints found for {self.model}.", "code": 404}}, 404)
         text, calls = reply_for(test, self.mode)
         self.send_response(200)
         self.send_header("Content-Type", "text/event-stream")
@@ -105,7 +108,7 @@ def main(argv=None):
     p = argparse.ArgumentParser()
     p.add_argument("--port", type=int, default=18090)
     p.add_argument("--packs", required=True)
-    p.add_argument("--mode", choices=("oracle", "wrong", "fixed"), default="oracle")
+    p.add_argument("--mode", choices=("oracle", "wrong", "fixed", "unavailable"), default="oracle")
     p.add_argument("--model", default="mock")
     p.add_argument("--delay", type=float, default=0.0, help="seconds each answer takes")
     a, _ = p.parse_known_args(argv)   # other flags (e.g. speed knobs under test) are accepted and ignored

{tuieval-0.2.0.dev42 → tuieval-0.2.0.dev43}/tests/test_tuieval.py

@@ -198,6 +198,16 @@ class WorkspaceGraders(unittest.TestCase):
 
 
 class Units(unittest.TestCase):
+    def test_server_errors_never_judge_the_model(self):
+        from tuieval import client
+        no_endpoints = 'server returned HTTP 404: {"error":{"message":"No endpoints found for some/model."}}'
+        for reason, retry, unavailable in ((no_endpoints, False, True), ("server returned HTTP 402: no credits", False, True),
+                                           ("server returned HTTP 503: busy", True, False),
+                                           ("server returned HTTP 400: bad request", False, False),
+                                           ("connection error: timed out", False, False)):   # the model too slow
+            self.assertEqual((client.is_server_error(reason), client.is_unavailable(reason)), (retry, unavailable), reason)
+            self.assertEqual(client.server_error_row({"pass": False, "reason": reason}), retry or unavailable, reason)
+
     def test_tune_workload_needs_no_packs(self):
         from tuieval import tune
         work = tune.workload(None, 65536)
@@ -787,6 +797,41 @@ class PTAIndex(unittest.TestCase):
             self.assertEqual([r[:4] for r in got["rows"]][0], ["good", "100", "100", "100"])
 
 
+class Unavailable(unittest.TestCase):
+    """A model no server will serve (OpenRouter's 404 "No endpoints found", a bad key, no credits) stops at
+    once and is never judged on it; the other models run as usual."""
+
+    def test_unavailable_model_stops_and_isnt_judged(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            ws = os.path.join(tmp, "ws")
+            tuieval(tmp, "init", ws)
+            d = os.path.join(ws, "packs", "apps")
+            os.makedirs(d)
+            write(os.path.join(d, "pack.toml"), 'label = "apps"\n[certify]\nrepeat = 1\n[gate]\nmin_accuracy = 0.1\n')
+            write(os.path.join(d, "tests.yaml"), "".join(
+                f"- {{id: {t}, input: question {t}, expected: 1, reference: 'ANSWER: 1', difficulty: easy}}\n"
+                for t in "abc"))
+            up, down = Mock(os.path.join(ws, "packs"), "oracle"), Mock(os.path.join(ws, "packs"), "unavailable")
+            try:
+                with open(os.path.join(ws, "models.toml"), "a") as f:
+                    f.write(f'\n[servers.up]\nurl = "http://127.0.0.1:{up.port}"\n'
+                            f'\n[servers.down]\nurl = "http://127.0.0.1:{down.port}"\n')
+                tuieval(ws, "add", "mock", "--server", "up", "--label", "works")
+                tuieval(ws, "add", "mock", "--server", "down", "--label", "gone")
+                t = time.time()
+                out = tuieval(ws, "run", "--tier", "certify", check=False).stdout
+            finally:
+                up.stop()
+                down.stop()
+            self.assertLess(time.time() - t, 60)                      # no retrying for minutes
+            self.assertIn("isn't available there", out)
+            verdict = tuieval(ws, "verdict", check=False).stdout
+            self.assertNotIn("0% right", verdict)                       # never judged on the 404s
+            rows = [l for l in verdict.splitlines() if "gone" in l]
+            self.assertFalse(any("FAIL" in l for l in rows), rows)
+            self.assertIn("works", tuieval(ws, "pta").stdout)
+
+
 class FirstRun(unittest.TestCase):
     """A new user's first run fails fast with the fix, never hangs or shows a bare Python error."""