tuieval 0.2.0.dev126__tar.gz → 0.2.0.dev127__tar.gz

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev127}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev126
+Version: 0.2.0.dev127
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev127}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev126'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev126')
+__version__ = version = '0.2.0.dev127'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev127')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev127}/tests/mock_server.py

@@ -56,6 +56,7 @@ class Handler(BaseHTTPRequestHandler):
     answers, mode, model, delay, max_output = {}, "oracle", "mock", 0.0, None
     stall, stall_before, silent = 0.0, 0.0, False
     die_after = None
+    hold, answered = None, 0   # while the hold file exists, answers after the first wait (a run that stays going)
 
     def log_message(self, *a):
         pass
@@ -120,6 +121,8 @@ class Handler(BaseHTTPRequestHandler):
                     self.wfile.flush()
                 time.sleep(0.2)
 
+        while self.hold and Handler.answered and os.path.exists(self.hold):
+            wait(0.2)
         wait(self.stall_before)
         send({"choices": [{"delta": {"reasoning_content": "Thinking it over. "}}]})
         wait(self.stall)
@@ -136,6 +139,7 @@ class Handler(BaseHTTPRequestHandler):
                           "prompt_per_second": 500.0, "cache_n": 0}})
         die_if_due()
         self.wfile.write(b"data: [DONE]\n\n")
+        Handler.answered += 1
 
 
 class Server(ThreadingHTTPServer):
@@ -157,11 +161,12 @@ def main(argv=None):
     p.add_argument("--stall-before", type=float, default=0.0)
     p.add_argument("--silent", action="store_true")
     p.add_argument("--die-after", type=int)
+    p.add_argument("--hold", help="while this file exists, every answer after the first waits")
     a, _ = p.parse_known_args(argv)   # other flags (e.g. speed knobs under test) are accepted and ignored
     Handler.answers, Handler.mode, Handler.model, Handler.delay = load_answers(a.packs), a.mode, a.model, a.delay
     Handler.max_output = a.max_output
     Handler.stall, Handler.stall_before, Handler.silent = a.stall, a.stall_before, a.silent
-    Handler.die_after = a.die_after
+    Handler.die_after, Handler.hold = a.die_after, a.hold
     srv = Server(("127.0.0.1", a.port), Handler)
     print(f"mock server on 127.0.0.1:{a.port} ({a.mode}, {len(Handler.answers)} known questions)", flush=True)
     try:

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev127}/tests/test_tuieval.py

@@ -1412,8 +1412,12 @@ class AddToRun(unittest.TestCase):
 
     def test_tui_never_queues_the_same_thing_twice(self):
         """Start while a run is going never adds or queues a model and pack already running or queued."""
+        hold = os.path.join(self.tmp.name, "hold")   # the first run stays going until every pick is made
+        write(hold, "")
+        path = os.path.join(self.ws, "models.toml")
+        write(path, read(path).replace('"--delay", "0.2"', f'"--delay", "0.2", "--hold", {json.dumps(hold)}'))
         code = textwrap.dedent("""
-            import asyncio, json, time
+            import asyncio, json, os, time
             from tuieval.tui import EvalsApp, RunScreen, SetupScreen, ChoiceScreen
             from textual.widgets import SelectionList
             async def pick(app, pilot, models, packs=("apps",)):
@@ -1459,6 +1463,7 @@ class AddToRun(unittest.TestCase):
                     if isinstance(s, ChoiceScreen):
                         s.dismiss("queue")
                     await pilot.pause()
+                    os.remove(os.environ["HOLD"])
                     deadline = time.time() + 300   # the queued runs start one after another: wait for all three
                     while time.time() < deadline:
                         await pilot.pause(0.2)
@@ -1469,7 +1474,7 @@ class AddToRun(unittest.TestCase):
             asyncio.run(go())
         """)
         p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=600,
-                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""))
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS="", HOLD=hold))
         out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
         self.assertIsInstance(out, dict, out)
         self.assertEqual(out["same"], [False, 0])