tuieval 0.2.0.dev3__py3-none-any.whl → 0.2.0.dev5__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev3'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev3')
+__version__ = version = '0.2.0.dev5'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev5')
 
 __commit_id__ = commit_id = None

tuieval/tune.py

@@ -26,7 +26,8 @@ An output guard compares greedy answers with the default flags; an option that c
 noise is rejected, because speed flags must not change answers. Servers whose answers depend on
 the machine's memory settings (`outputs_depend_on_machine`) skip the guard; their tuned flags
 become part of the results fingerprint instead, so retuning marks that machine's results
-outdated. Any candidate that makes macOS swap is rejected. Servers without tune knobs are only
+outdated. Any candidate under which macOS swaps while the server works is rejected (swapping
+while the model loads, e.g. to lock it in RAM, is only noted). Servers without tune knobs are only
 measured.
 
 The knobs and their options come from models.toml [servers.<name>.tune]; placeholders {p},
@@ -315,8 +316,11 @@ def family(path):
         i = machines.read_gguf(engine_mod.expand(path))
     except (OSError, ValueError):
         return None
+    # The MTP (nextn) draft layers at the end are left out: GGUFs of the same model describe them
+    # differently, and whether draft-mtp is offered is decided per model (knobs) anyway.
+    heads = i["kv_heads_per_layer"][:len(i["kv_heads_per_layer"]) - i["mtp_layers"]]
     return (i["architecture"], i["layers"], i["embedding"], i["experts"], i["experts_used"],
-            i["head_dim_k"], i["head_dim_v"], tuple(i["kv_heads_per_layer"]))
+            i["head_dim_k"], i["head_dim_v"], tuple(heads))
 
 
 @dataclasses.dataclass
@@ -471,12 +475,25 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         emit("tune_step", message=f"[{starts[0]}] {why}: {' '.join(args) or '(server defaults)'}",
              start=starts[0], max_starts=max_starts)
         swap0 = machines.swapped_out_bytes()
+        swapped = None   # MB swapped out while the loaded server worked (loading itself doesn't count)
         try:
             with eng.serve(m, perf_args=fixed + args, log_name=f"{label}.tune") as (url, info):
+                # Loading may push other apps to swap once (e.g. a model locked in RAM with mlock);
+                # that's recorded, not held against the settings. Swapping while serving is.
+                loaded = machines.swapped_out_bytes()
                 r = measure(eng, m, url, work, gen_tokens, passes, emit,
                             settings.get("request_timeout_s", REQUEST_LIMIT_S))
+                end = machines.swapped_out_bytes()
+                if None not in (swap0, loaded, end):
+                    swapped = (end - loaded) / 2**20
+                    at_load = (loaded - swap0) / 2**20
+                    if at_load > SWAP_LIMIT / 2**20:
+                        emit("tune_step", message=f"    note: loading pushed {at_load:.0f} MB of other apps to swap "
+                                                  "(close apps for more headroom)")
                 r.load_s = round(info["load_s"], 1) if info["load_s"] else None
                 r.facts = dict(info["facts"])
+                if swapped is not None and at_load > SWAP_LIMIT / 2**20:
+                    r.facts["load_swapped_mb"] = round(at_load)
                 # Settled allocation after the timed pass (brief peaks while processing a prompt are
                 # harmless; sustained allocation over the limit is what makes the driver churn).
                 settled = machines.gpu_allocated_gb() if server.get("stall_guard") else None
@@ -494,9 +511,8 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                                               "close other apps for faster and fairer results")
         except engine_mod.ModelFailed as e:
             r = Measure(error=str(e).splitlines()[0])
-        swap1 = machines.swapped_out_bytes()
-        if not r.error and swap0 is not None and swap1 is not None and swap1 - swap0 > SWAP_LIMIT:
-            r.error = f"macOS swapped {(swap1 - swap0) / 2**20:.0f} MB (memory too tight)"
+        if not r.error and swapped is not None and swapped > SWAP_LIMIT / 2**20:
+            r.error = f"macOS swapped {swapped:.0f} MB while the server worked (memory too tight)"
         cache[k] = r
         emit("tune_result", message=f"    failed: {r.error}" if r.error else "    " + r.summary(), result=r, args=args)
         return r

{tuieval-0.2.0.dev3.dist-info → tuieval-0.2.0.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev3
+Version: 0.2.0.dev5
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev3.dist-info → tuieval-0.2.0.dev5.dist-info}/RECORD

@@ -1,6 +1,6 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=2ILWlstNdKCO-w6pi6qFsd7f8ZJMAD8pn_6jJ7S_lmY,533
+tuieval/_version.py,sha256=s3bhRhA0S1Yy9V9RoQJoCTBpd5_-bYzuMi3GCnt_F5k,533
 tuieval/cli.py,sha256=ZaLoWrxyCABu4SJzx5U2gVUUmwcwRgpZVqdqnxyFMOk,4167
 tuieval/client.py,sha256=Vui7ucVv_GiKObGoVS_EWSZFUKPou9PWmutqMTrahXk,9313
 tuieval/compare.py,sha256=b-rVcMwq0Tos3oNKcpUP1HXULKhAQNAS4rlXYSY7t6k,25460
@@ -14,7 +14,7 @@ tuieval/run_evals.py,sha256=uPzidbRPUduBwP8sKyzg-U8cka0p-RsbZlVis6qQTlk,37526
 tuieval/scaffold.py,sha256=-AE5vKXeTtMZJ8OMk4UiIstoUWozTKjThKxZ87j0f9E,4720
 tuieval/selftest.py,sha256=xKfR73wMTYcDRSCBYQTNyKeR9bYU5HIYUSg7oZvNCyI,9330
 tuieval/tui.py,sha256=8diKtYE31ZUKq62rfMGdMkEbHgK6arRYpjFu-wcMVZs,131997
-tuieval/tune.py,sha256=zOs1cfKS3NhaFenjCY6fDb7oa8cx5lsoxgGaIEas_ys,32625
+tuieval/tune.py,sha256=dPVs_l4xBpq0ymjreAvcMAk1QmPVazf9_tLxooGoY88,33876
 tuieval/verdict.py,sha256=oLKpRIruk7W8eF1YdmITFx9BgBK3kPKYViKL20Nzyu0,16306
 tuieval/watch_proxy.py,sha256=z916NRhEur04a9RnLm6ZpIEY1SnnUGPiHYbmlD9ktQ0,11035
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
@@ -42,8 +42,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev3.dist-info/METADATA,sha256=xZqfkjw1ZXBPMF-goFno6NYjbKpP379DrEAcKfZC2fg,12079
-tuieval-0.2.0.dev3.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev3.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev3.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev3.dist-info/RECORD,,
+tuieval-0.2.0.dev5.dist-info/METADATA,sha256=_-BmKuon4FeyODFRw63PSgzIPfsziCzapadFDb349ps,12079
+tuieval-0.2.0.dev5.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev5.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev5.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev5.dist-info/RECORD,,