tuieval 0.2.0.dev99__py3-none-any.whl → 0.2.0.dev100__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev99'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev99')
+__version__ = version = '0.2.0.dev100'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev100')
 
 __commit_id__ = commit_id = None

tuieval/compare.py

@@ -41,6 +41,7 @@ def _load_native(data):
             "pack": pack["name"], "pack_fp": pack["fingerprint"],
             # hosted (OpenRouter) models: the provider endpoint and its declared quantization
             "endpoint": run.get("endpoint"), "quantization": run.get("quantization"),
+            "hosted": bool(run.get("hosted") or run.get("endpoint")),
             "loaded_alongside": run.get("loaded_alongside") or []}
     rows = []
     for r in data["results"]:
@@ -78,9 +79,37 @@ def load(paths):
             row.update(raw=rec, path=path, pack_fp=data["pack"]["fingerprint"])
         rows += r
         infos.append(i)
+    # older results also noted hosted models alongside, which share nothing of this machine: a hosted
+    # model's rows note no one, and no one notes it
+    labels, prefixes = configured_hosted(paths)
+    labels |= {i["label"] for i in infos if i["hosted"]}
+
+    def local(along):
+        return [l for l in along if l not in labels and not l.startswith(tuple(prefixes))]
+    for row in rows:
+        row["alongside"] = [] if row["model"] in labels else local(row["alongside"])
+    for i in infos:
+        i["loaded_alongside"] = [] if i["label"] in labels else local(i["loaded_alongside"])
     return rows, infos
 
 
+def configured_hosted(paths):
+    """(labels, label prefixes) models.toml says are hosted, in the workspaces these results/<label>/
+    <pack>.json are from: its hosted models, and the prefix of models named at run time on a hosted
+    server (e.g. "or-"). Covers hosted models whose results aren't loaded."""
+    from . import engine
+    labels, prefixes = set(), set()
+    for root in {os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(p)))) for p in paths}:
+        try:
+            cfg = engine.load_config(os.path.join(root, "models.toml"))
+        except Exception:
+            continue    # no readable models.toml there: the results' own marks still apply
+        labels |= {m["label"] for m in cfg["models"] if engine.hosted(cfg, m)}
+        prefixes |= {engine.remote_label(n, s, "") for n, s in cfg["servers"].items()
+                     if s.get("any_model") and engine.hosted(cfg, {"server": n})}
+    return labels, prefixes
+
+
 def default_paths(results_dir=None):
     """results/<label>/<pack>.json, excluding smoke runs and the archive."""
     results_dir = results_dir or workspace.path("results")

tuieval/engine.py

@@ -83,6 +83,21 @@ def builtin_config():
         return tomllib.load(f)
 
 
+def hosted(cfg, m):
+    """A model served by an API elsewhere: tuieval doesn't start its server and its url isn't this
+    machine (or its server sets hosted = true). A server you started yourself on localhost still
+    counts as local (it uses the GPU)."""
+    if m.get("remote"):
+        return True
+    server = cfg["servers"][m["server"]]
+    if "hosted" in server:   # say so for a server the url doesn't give away (e.g. a local proxy to an API)
+        return bool(server["hosted"])
+    if server.get("cmd"):
+        return False
+    host = urllib.parse.urlparse(server.get("url") or "").hostname or ""
+    return bool(host) and host not in ("localhost", "127.0.0.1", "::1", "0.0.0.0")
+
+
 def load_config(path):
     """Read models.toml, fill in the built-ins it leaves out (builtin.toml) and each model's defaults.
     [defaults] are filled key by key; [sampling] only when models.toml has none (sampling is part of
@@ -675,8 +690,9 @@ class Lane:
     """One model being run: its own request stream, stall reason and skip switch, so models run
     side by side (parallel_models) stop independently. Runs and tunes in one lane use the engine's
     main lane."""
-    def __init__(self, label=None):
+    def __init__(self, label=None, hosted=False):
         self.label = label
+        self.hosted = hosted      # a hosted model shares nothing of this machine, so it's never noted alongside
         self.stream = None
         self.stall = None         # set by the stall watcher: why this model's server stalled
         self.skip = threading.Event()
@@ -1282,18 +1298,7 @@ class Engine:
                 + min(parallel_hosted or self.parallel_hosted(), hosted))
 
     def hosted(self, m):
-        """A model served by an API elsewhere: tuieval doesn't start its server and its url isn't this
-        machine (or its server sets hosted = true). A server you started yourself on localhost still
-        counts as local (it uses the GPU)."""
-        if m.get("remote"):
-            return True
-        server = self.cfg["servers"][m["server"]]
-        if "hosted" in server:   # say so for a server the url doesn't give away (e.g. a local proxy to an API)
-            return bool(server["hosted"])
-        if server.get("cmd"):
-            return False
-        host = urllib.parse.urlparse(server.get("url") or "").hostname or ""
-        return bool(host) and host not in ("localhost", "127.0.0.1", "::1", "0.0.0.0")
+        return hosted(self.cfg, m)
 
     def memory_need_gb(self, m):
         """GPU memory a model takes while served here: 0 for one we don't start (a hosted API or a
@@ -1817,15 +1822,23 @@ class Engine:
         for t in threads:
             t.join()
 
+    def _local_lanes(self, lane, exclude=None):
+        """Labels of the local models served now (call with _lanes_lock held); none for a hosted lane."""
+        if lane.hosted:
+            return set()
+        return {l for l, o in self._lanes.items() if not o.hosted} - {exclude}
+
     def _run_lane(self, label, mine):
         """One model's jobs, in this thread's own lane. Other models served meanwhile are noted on
-        each answer (ran_alongside), since they share the machine's GPU and memory bandwidth."""
-        lane = Lane(label)
+        each answer (ran_alongside), since they share the machine's GPU and memory bandwidth. Hosted
+        models share neither, so they note nothing and nobody notes them."""
         m = mine[0].model
+        lane = Lane(label, self.hosted(m))
         with self._lanes_lock:
-            for other in self._lanes.values():
-                other.seen.add(label)
-            lane.seen = set(self._lanes)
+            if not lane.hosted:
+                for other in self._lanes.values():
+                    other.seen.add(label)
+            lane.seen = self._local_lanes(lane)
             if self.cfg["servers"][m["server"]].get("cmd"):
                 lane.port = self._lane_port(m)
             self._lanes[label] = lane
@@ -1987,6 +2000,8 @@ class Engine:
                         "kv_type": sv.kv_type, "perf_args": sv.perf_args if local else None,
                         "perf_source": sv.perf_source, "server_version": self.server_version(m) if local else None}
         ep = self.endpoint(m)
+        if self.hosted(m):
+            serving_info["hosted"] = True
         if ep:
             serving_info.update(endpoint=ep["tag"], provider=ep.get("provider"), quantization=ep.get("quantization"))
         for j in jobs:
@@ -2244,7 +2259,7 @@ class Engine:
                 self._check()
                 lane = self._lane()
                 with self._lanes_lock:
-                    lane.seen = set(self._lanes) - {job.label}
+                    lane.seen = self._local_lanes(lane, job.label)
                 self.emit("request_started", job=job, test=test["description"], test_id=test["id"],
                           has_image=bool(test.get("image")), repeat=rep, difficulty=test.get("difficulty", "unrated"))
                 waits = list(self.SERVER_RETRY_WAITS_S)

{tuieval-0.2.0.dev99.dist-info → tuieval-0.2.0.dev100.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev99
+Version: 0.2.0.dev100
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev99.dist-info → tuieval-0.2.0.dev100.dist-info}/RECORD

@@ -1,12 +1,12 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=EnjLNHE3-O1-hviWXp7HEQ3pbBMWrRtAx08ValpcMyA,535
+tuieval/_version.py,sha256=y3vaJ3eXdQYaUkE0tzZXiiT1vRZ_h4vmv3rHfxsmx3g,537
 tuieval/builtin.toml,sha256=Nwu7O0qFzcrDAxXO_VZtDPvjVwxZ29OtYOW0BPihZGs,4410
 tuieval/cli.py,sha256=mr0o_ud6NctZeU4WIrh17x_uzEK0vTeCynGP0wrCeBY,4812
 tuieval/client.py,sha256=wtACBVsngtWopWWhIc4bB6mEUEi31dvvDePN-PjmXbM,11122
-tuieval/compare.py,sha256=uXU_Yzf4zDSHoupHQrTmBhsfc8wCyI1nMaZQN81ybiA,30522
+tuieval/compare.py,sha256=UjZcWsbPep7EG4XRKWaEthUT0l0iN8rKIs1135dbt6c,32139
 tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
-tuieval/engine.py,sha256=96GWzKzMQ_KaG6wXrIuBiKWFCcblXWL9cQs89QYgj7Y,125256
+tuieval/engine.py,sha256=cD9N5TxK0XKrGYm5G0aEY3FYqU4tY36didaH-MYU788,125887
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/insights.py,sha256=m6HZk1VbX-avHr3xyT8eVJ5OLjKa_EKTVBxsu04X_HQ,28947
 tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
@@ -48,8 +48,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev99.dist-info/METADATA,sha256=ulrRF0F1nVj2zmY3ds6cF6VpvOBOG3rOOjbZeyNxIT0,14580
-tuieval-0.2.0.dev99.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev99.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev99.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev99.dist-info/RECORD,,
+tuieval-0.2.0.dev100.dist-info/METADATA,sha256=Fi08-nci1K46vMk8ohZOF3ztQt2d_ud2INMG3dYDcF4,14581
+tuieval-0.2.0.dev100.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev100.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev100.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev100.dist-info/RECORD,,