tuieval 0.2.0.dev50__tar.gz → 0.2.0.dev55__tar.gz

{tuieval-0.2.0.dev50 → tuieval-0.2.0.dev55}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Faster start on Macs: the GPU core count comes from `ioreg` instead of `system_profiler`, which took seconds on some Macs.
 - A model a server won't serve (HTTP 401-404, e.g. OpenRouter's "No endpoints found") is never judged on it: those answers don't count, and the model stops at once instead of failing every question.
 - PTA index: parsimony (tokens used) and speed (time), each 100 for the best and 15 points less per doubling, and accuracy per model, as bars: the first Results tab (where `r` lands), `tuieval pta` and the report. Server errors aren't answers, and models with few answers are left out unless picked.
 - Run several models at a time: `parallel_models` per machine, **Models at a time** in the TUI, or `tuieval run --parallel N` (default 1). Models wait for memory to fit, and answers note what ran alongside them.

{tuieval-0.2.0.dev50 → tuieval-0.2.0.dev55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev50
+Version: 0.2.0.dev55
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev50 → tuieval-0.2.0.dev55}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev50'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev50')
+__version__ = version = '0.2.0.dev55'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev55')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev50 → tuieval-0.2.0.dev55}/src/tuieval/machines.py

@@ -65,10 +65,10 @@ def detect():
         # macOS default when unset: about 2/3 of RAM up to 36 GB, 3/4 above.
         limit = wired / 1024 if wired > 0 else ram * (0.75 if ram > 36 else 2 / 3)
         gpu_cores = None
-        try:
-            out = subprocess.run(["system_profiler", "SPDisplaysDataType"], capture_output=True, text=True,
-                                 timeout=15).stdout
-            m = re.search(r"Total Number of Cores:\s*(\d+)", out)
+        try:   # the GPU driver's own count: instant, where system_profiler can take seconds (CI's virtual Macs)
+            out = subprocess.run(["ioreg", "-rc", "AGXAccelerator", "-d1"], capture_output=True, text=True,
+                                 timeout=5).stdout
+            m = re.search(r'"gpu-core-count"\s*=\s*(\d+)', out)
             gpu_cores = int(m.group(1)) if m else None
         except (OSError, subprocess.TimeoutExpired):
             pass

{tuieval-0.2.0.dev50 → tuieval-0.2.0.dev55}/src/tuieval/watch_proxy.py

@@ -18,6 +18,7 @@ import http.client
 import itertools
 import json
 import os
+import socketserver
 import sys
 import threading
 import time
@@ -129,6 +130,12 @@ class Proxy:
         class Server(ThreadingHTTPServer):
             daemon_threads = True
 
+            def server_bind(self):
+                # HTTPServer looks up its own hostname here (socket.getfqdn), which can take half a
+                # minute where DNS is slow; the name is only cosmetic, so use the address.
+                socketserver.TCPServer.server_bind(self)
+                self.server_name, self.server_port = self.server_address[:2]
+
             def handle_error(self, request, client_address):
                 err = sys.exc_info()[1]
                 if isinstance(err, (BrokenPipeError, ConnectionResetError)):

{tuieval-0.2.0.dev50 → tuieval-0.2.0.dev55}/tests/mock_server.py

@@ -15,6 +15,7 @@ import json
 import os
 import sys
 import time
+import socketserver
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 import yaml
@@ -104,6 +105,13 @@ class Handler(BaseHTTPRequestHandler):
         self.wfile.write(b"data: [DONE]\n\n")
 
 
+class Server(ThreadingHTTPServer):
+    def server_bind(self):
+        # skip HTTPServer's own-hostname lookup (socket.getfqdn): 35 s per server on CI's macOS runners
+        socketserver.TCPServer.server_bind(self)
+        self.server_name, self.server_port = self.server_address[:2]
+
+
 def main(argv=None):
     p = argparse.ArgumentParser()
     p.add_argument("--port", type=int, default=18090)
@@ -113,7 +121,7 @@ def main(argv=None):
     p.add_argument("--delay", type=float, default=0.0, help="seconds each answer takes")
     a, _ = p.parse_known_args(argv)   # other flags (e.g. speed knobs under test) are accepted and ignored
     Handler.answers, Handler.mode, Handler.model, Handler.delay = load_answers(a.packs), a.mode, a.model, a.delay
-    srv = ThreadingHTTPServer(("127.0.0.1", a.port), Handler)
+    srv = Server(("127.0.0.1", a.port), Handler)
     print(f"mock server on 127.0.0.1:{a.port} ({a.mode}, {len(Handler.answers)} known questions)", flush=True)
     try:
         srv.serve_forever()