tuieval 0.2.0.dev99__tar.gz → 0.2.0.dev100__tar.gz

{tuieval-0.2.0.dev99 → tuieval-0.2.0.dev100}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix: hosted models no longer count as running alongside local ones (they don't share the machine), so a one-model-at-a-time setup no longer gets the "ran alongside other models" note. Older results are read the same way.
 - Results, Per question: the model summary shows every picked model without scrolling, and fits the window: shorter column names, and long model names cut in the middle (both ends kept). The PTA index table fits the same way.
 - PTA index: the accuracy bar shows answers cut off at max_tokens in orange, with `(n% truncated)` beside it and a **truncated** column in the table, so overthinking stands apart from wrong answers.
 - Results → Insights (also `tuieval insights`): charts with what to do next: every model against a reference (95% CI), the best model per use case, fine-tunes against their base (`base` in models.toml), why models fail, accuracy against tokens or time, question health, and the Certify runs that would settle the strongest models' verdicts. `[defaults] reference` sets the default reference.

{tuieval-0.2.0.dev99 → tuieval-0.2.0.dev100}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev99
+Version: 0.2.0.dev100
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev99 → tuieval-0.2.0.dev100}/docs/models.md

@@ -94,7 +94,7 @@ By default a run serves one model at a time. On a machine with room for more, se
 - Models start in queue order, each in its own lane. The next one waits while its fit-check memory estimate wouldn't fit next to the ones running (without one, e.g. a server that sizes its own memory, the setting decides). Contexts are never shrunk to make room.
 - A second model on the same server gets a free port, so a server's `cmd` must take `{port}`.
 - A server with `before_start` (which may stop other servers) always runs on its own. Tuning is always one model at a time.
-- Answers are judged exactly as in a run of one model. Each answer records the models served while it ran (`ran_alongside`), and a load time measured next to others is marked (`loaded_alongside`), since models side by side share the machine's GPU and memory bandwidth. The speed table and the answer view show it.
+- Answers are judged exactly as in a run of one model. Each answer records the models served while it ran (`ran_alongside`), and a load time measured next to others is marked (`loaded_alongside`), since models side by side share the machine's GPU and memory bandwidth. Only local models count: hosted ones share nothing of the machine. The speed table and the answer view show it.
 - In the TUI, one model's answers stream at a time: `v` switches to the next, `k` skips the one streaming. `tuieval run` prints one line per answer instead of streaming.
 - Only one run uses the machine at a time across tuieval windows; models at a time applies within a run.
 

{tuieval-0.2.0.dev99 → tuieval-0.2.0.dev100}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev99'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev99')
+__version__ = version = '0.2.0.dev100'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev100')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev99 → tuieval-0.2.0.dev100}/src/tuieval/compare.py

@@ -41,6 +41,7 @@ def _load_native(data):
             "pack": pack["name"], "pack_fp": pack["fingerprint"],
             # hosted (OpenRouter) models: the provider endpoint and its declared quantization
             "endpoint": run.get("endpoint"), "quantization": run.get("quantization"),
+            "hosted": bool(run.get("hosted") or run.get("endpoint")),
             "loaded_alongside": run.get("loaded_alongside") or []}
     rows = []
     for r in data["results"]:
@@ -78,9 +79,37 @@ def load(paths):
             row.update(raw=rec, path=path, pack_fp=data["pack"]["fingerprint"])
         rows += r
         infos.append(i)
+    # older results also noted hosted models alongside, which share nothing of this machine: a hosted
+    # model's rows note no one, and no one notes it
+    labels, prefixes = configured_hosted(paths)
+    labels |= {i["label"] for i in infos if i["hosted"]}
+
+    def local(along):
+        return [l for l in along if l not in labels and not l.startswith(tuple(prefixes))]
+    for row in rows:
+        row["alongside"] = [] if row["model"] in labels else local(row["alongside"])
+    for i in infos:
+        i["loaded_alongside"] = [] if i["label"] in labels else local(i["loaded_alongside"])
     return rows, infos
 
 
+def configured_hosted(paths):
+    """(labels, label prefixes) models.toml says are hosted, in the workspaces these results/<label>/
+    <pack>.json are from: its hosted models, and the prefix of models named at run time on a hosted
+    server (e.g. "or-"). Covers hosted models whose results aren't loaded."""
+    from . import engine
+    labels, prefixes = set(), set()
+    for root in {os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(p)))) for p in paths}:
+        try:
+            cfg = engine.load_config(os.path.join(root, "models.toml"))
+        except Exception:
+            continue    # no readable models.toml there: the results' own marks still apply
+        labels |= {m["label"] for m in cfg["models"] if engine.hosted(cfg, m)}
+        prefixes |= {engine.remote_label(n, s, "") for n, s in cfg["servers"].items()
+                     if s.get("any_model") and engine.hosted(cfg, {"server": n})}
+    return labels, prefixes
+
+
 def default_paths(results_dir=None):
     """results/<label>/<pack>.json, excluding smoke runs and the archive."""
     results_dir = results_dir or workspace.path("results")

{tuieval-0.2.0.dev99 → tuieval-0.2.0.dev100}/src/tuieval/engine.py

@@ -83,6 +83,21 @@ def builtin_config():
         return tomllib.load(f)
 
 
+def hosted(cfg, m):
+    """A model served by an API elsewhere: tuieval doesn't start its server and its url isn't this
+    machine (or its server sets hosted = true). A server you started yourself on localhost still
+    counts as local (it uses the GPU)."""
+    if m.get("remote"):
+        return True
+    server = cfg["servers"][m["server"]]
+    if "hosted" in server:   # say so for a server the url doesn't give away (e.g. a local proxy to an API)
+        return bool(server["hosted"])
+    if server.get("cmd"):
+        return False
+    host = urllib.parse.urlparse(server.get("url") or "").hostname or ""
+    return bool(host) and host not in ("localhost", "127.0.0.1", "::1", "0.0.0.0")
+
+
 def load_config(path):
     """Read models.toml, fill in the built-ins it leaves out (builtin.toml) and each model's defaults.
     [defaults] are filled key by key; [sampling] only when models.toml has none (sampling is part of
@@ -675,8 +690,9 @@ class Lane:
     """One model being run: its own request stream, stall reason and skip switch, so models run
     side by side (parallel_models) stop independently. Runs and tunes in one lane use the engine's
     main lane."""
-    def __init__(self, label=None):
+    def __init__(self, label=None, hosted=False):
         self.label = label
+        self.hosted = hosted      # a hosted model shares nothing of this machine, so it's never noted alongside
         self.stream = None
         self.stall = None         # set by the stall watcher: why this model's server stalled
         self.skip = threading.Event()
@@ -1282,18 +1298,7 @@ class Engine:
                 + min(parallel_hosted or self.parallel_hosted(), hosted))
 
     def hosted(self, m):
-        """A model served by an API elsewhere: tuieval doesn't start its server and its url isn't this
-        machine (or its server sets hosted = true). A server you started yourself on localhost still
-        counts as local (it uses the GPU)."""
-        if m.get("remote"):
-            return True
-        server = self.cfg["servers"][m["server"]]
-        if "hosted" in server:   # say so for a server the url doesn't give away (e.g. a local proxy to an API)
-            return bool(server["hosted"])
-        if server.get("cmd"):
-            return False
-        host = urllib.parse.urlparse(server.get("url") or "").hostname or ""
-        return bool(host) and host not in ("localhost", "127.0.0.1", "::1", "0.0.0.0")
+        return hosted(self.cfg, m)
 
     def memory_need_gb(self, m):
         """GPU memory a model takes while served here: 0 for one we don't start (a hosted API or a
@@ -1817,15 +1822,23 @@ class Engine:
         for t in threads:
             t.join()
 
+    def _local_lanes(self, lane, exclude=None):
+        """Labels of the local models served now (call with _lanes_lock held); none for a hosted lane."""
+        if lane.hosted:
+            return set()
+        return {l for l, o in self._lanes.items() if not o.hosted} - {exclude}
+
     def _run_lane(self, label, mine):
         """One model's jobs, in this thread's own lane. Other models served meanwhile are noted on
-        each answer (ran_alongside), since they share the machine's GPU and memory bandwidth."""
-        lane = Lane(label)
+        each answer (ran_alongside), since they share the machine's GPU and memory bandwidth. Hosted
+        models share neither, so they note nothing and nobody notes them."""
         m = mine[0].model
+        lane = Lane(label, self.hosted(m))
         with self._lanes_lock:
-            for other in self._lanes.values():
-                other.seen.add(label)
-            lane.seen = set(self._lanes)
+            if not lane.hosted:
+                for other in self._lanes.values():
+                    other.seen.add(label)
+            lane.seen = self._local_lanes(lane)
             if self.cfg["servers"][m["server"]].get("cmd"):
                 lane.port = self._lane_port(m)
             self._lanes[label] = lane
@@ -1987,6 +2000,8 @@ class Engine:
                         "kv_type": sv.kv_type, "perf_args": sv.perf_args if local else None,
                         "perf_source": sv.perf_source, "server_version": self.server_version(m) if local else None}
         ep = self.endpoint(m)
+        if self.hosted(m):
+            serving_info["hosted"] = True
         if ep:
             serving_info.update(endpoint=ep["tag"], provider=ep.get("provider"), quantization=ep.get("quantization"))
         for j in jobs:
@@ -2244,7 +2259,7 @@ class Engine:
                 self._check()
                 lane = self._lane()
                 with self._lanes_lock:
-                    lane.seen = set(self._lanes) - {job.label}
+                    lane.seen = self._local_lanes(lane, job.label)
                 self.emit("request_started", job=job, test=test["description"], test_id=test["id"],
                           has_image=bool(test.get("image")), repeat=rep, difficulty=test.get("difficulty", "unrated"))
                 waits = list(self.SERVER_RETRY_WAITS_S)

{tuieval-0.2.0.dev99 → tuieval-0.2.0.dev100}/tests/test_tuieval.py

@@ -893,15 +893,42 @@ class HostedBeside(unittest.TestCase):
 
     def test_hosted_run_beside_local_with_one_slot(self):
         from tuieval import engine
-        e = engine.Engine(lambda kind, **d: None, root=self.ws)
+        together = set()
+
+        def on_event(kind, **d):
+            if kind == "request_started":
+                together.add(frozenset(e._lanes))
+        e = engine.Engine(on_event, root=self.ws)
         self.assertEqual([e.hosted(e.model(l)) for l in ("loc", "h1", "h2")], [False, True, True])
         self.assertEqual(e.models_at_once({"loc", "h1", "h2"}, 1, 4), 3)
         jobs = e.plan(["loc", "h1", "h2"], ["apps"], None, "certify", False, {})
         e.run(jobs, parallel=1)
         self.assertEqual([j.status for j in jobs], ["done"] * 3, [(j.key, j.note) for j in jobs])
-        seen = {l: set().union(*(r.get("ran_alongside", []) for r in self.results(l))) for l in ("loc", "h1", "h2")}
-        self.assertEqual(seen["loc"], {"h1", "h2"})          # neither hosted model waited for the local one
-        self.assertIn("h2", seen["h1"])
+        self.assertIn(frozenset({"loc", "h1", "h2"}), together)   # neither hosted model waited for the local one
+        # hosted models share nothing of this machine: no one notes them alongside, and they note no one
+        for label in ("loc", "h1", "h2"):
+            self.assertFalse(any("ran_alongside" in r for r in self.results(label)), label)
+        data = json.loads(read(os.path.join(self.ws, "results", "h1", "apps.json")))
+        self.assertTrue(data["run"]["hosted"])
+        self.assertNotIn("loaded_alongside", json.loads(read(os.path.join(self.ws, "results", "loc", "apps.json")))["run"])
+
+    def test_older_results_drop_hosted_models_alongside(self):
+        from tuieval import compare
+        paths = []
+        # h1's results say it's hosted; or-gone (named at run time, results deleted) by its server's prefix
+        for label, run, along in (("loc", {}, ["h1", "loc2", "or-gone"]), ("h1", {"endpoint": "x/fp8"}, ["loc"])):
+            data = {"format": "evals/1", "model": {"label": label}, "pack": {"name": "p", "fingerprint": "f"},
+                    "run": {**run, "loaded_alongside": along}, "settings": {},
+                    "results": [{"test": "t", "description": "d", "pass": True, "score": 1.0, "reason": "",
+                                 "ran_alongside": along}]}
+            path = os.path.join(self.ws, "results", label, "p.json")
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+            with open(path, "w") as f:
+                json.dump(data, f)
+            paths.append(path)
+        rows, infos = compare.load(paths)
+        self.assertEqual({r["model"]: r["alongside"] for r in rows}, {"loc": ["loc2"], "h1": []})
+        self.assertEqual({i["label"]: i["loaded_alongside"] for i in infos}, {"loc": ["loc2"], "h1": []})
 
     def test_local_model_joins_a_hosted_only_run(self):
         from tuieval import engine