tuieval 0.2.0.dev2__py3-none-any.whl → 0.2.0.dev4__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev2'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev2')
+__version__ = version = '0.2.0.dev4'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev4')
 
 __commit_id__ = commit_id = None

tuieval/engine.py

@@ -96,6 +96,16 @@ def load_config(path):
     return cfg
 
 
+def fit_check(server):
+    """Whether tuieval sizes the context for this server's models from GGUF headers (machines.fit,
+    which assumes llama.cpp's memory use). By default only servers whose command takes {ctx}, i.e.
+    where tuieval chooses the context; others size their own memory, and their context is the
+    model's max_context. models.toml `fit_check = true|false` on a server overrides it."""
+    if "fit_check" in server:
+        return bool(server["fit_check"])
+    return any("{ctx}" in str(a) for a in server.get("cmd", []))
+
+
 def effective_sampling(cfg, m):
     """The request settings this model runs with: [sampling], then the model's own overrides."""
     s = dict(cfg["sampling"])
@@ -1018,13 +1028,15 @@ class Engine:
         path = expand(m["model"])
         if server.get("model_is_path") and os.path.isfile(path):
             try:
-                mmproj = expand(m.get("mmproj", ""))
-                extra = os.path.getsize(mmproj) if mmproj and os.path.isfile(mmproj) else 0
-                headroom = self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
-                f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=headroom, extra_bytes=extra,
-                                 want_ctx=want)
                 info = machines.read_gguf(path)
-                ctx, fits, note, mtp, size = f.max_ctx, f.fits, f.note, info["mtp_layers"], info["bytes"]
+                mtp, size = info["mtp_layers"], info["bytes"]
+                if fit_check(server):
+                    mmproj = expand(m.get("mmproj", ""))
+                    extra = os.path.getsize(mmproj) if mmproj and os.path.isfile(mmproj) else 0
+                    headroom = self.machine_settings(mc.id).get("memory_headroom_gb", 4.0)
+                    f = machines.fit(path, mc, kv_type=kv or "f16", headroom_gb=headroom, extra_bytes=extra,
+                                     want_ctx=want)
+                    ctx, fits, note = f.max_ctx, f.fits, f.note
             except (OSError, ValueError) as e:
                 note = f"couldn't read the GGUF header ({e}); context not checked"
         profile, perf, source = None, [], "n/a"

tuieval/tune.py

@@ -315,8 +315,11 @@ def family(path):
         i = machines.read_gguf(engine_mod.expand(path))
     except (OSError, ValueError):
         return None
+    # The MTP (nextn) draft layers at the end are left out: GGUFs of the same model describe them
+    # differently, and whether draft-mtp is offered is decided per model (knobs) anyway.
+    heads = i["kv_heads_per_layer"][:len(i["kv_heads_per_layer"]) - i["mtp_layers"]]
     return (i["architecture"], i["layers"], i["embedding"], i["experts"], i["experts_used"],
-            i["head_dim_k"], i["head_dim_v"], tuple(i["kv_heads_per_layer"]))
+            i["head_dim_k"], i["head_dim_v"], tuple(heads))
 
 
 @dataclasses.dataclass

{tuieval-0.2.0.dev2.dist-info → tuieval-0.2.0.dev4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev2
+Version: 0.2.0.dev4
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev2.dist-info → tuieval-0.2.0.dev4.dist-info}/RECORD

@@ -1,10 +1,10 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=EOZVyFNK9VEmUe6471HI7YHfwqJ5uDi2cXJHVylN6eI,533
+tuieval/_version.py,sha256=7rQfE9HWjmke7Ju8Nk_aqQp4chEDuGAHhvH3RPUbSyc,533
 tuieval/cli.py,sha256=ZaLoWrxyCABu4SJzx5U2gVUUmwcwRgpZVqdqnxyFMOk,4167
 tuieval/client.py,sha256=Vui7ucVv_GiKObGoVS_EWSZFUKPou9PWmutqMTrahXk,9313
 tuieval/compare.py,sha256=b-rVcMwq0Tos3oNKcpUP1HXULKhAQNAS4rlXYSY7t6k,25460
-tuieval/engine.py,sha256=Amft1EL0iQVsiutfN7vQpwRXMCq_TYryDj0pMtY2suM,94195
+tuieval/engine.py,sha256=SizirCzV1klc_a2nS14FIjS5jLiHCGbVT23eES4StEc,94817
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/machines.py,sha256=CeestY-_6h8ZKhJbBA_xyf0prqBvjvdgs2XJtCJYnNg,12974
 tuieval/packs.py,sha256=Go-kTSPvYZriXMo4DvmqNNL-iD0J92MKLiNCnP2chJ8,10305
@@ -14,7 +14,7 @@ tuieval/run_evals.py,sha256=uPzidbRPUduBwP8sKyzg-U8cka0p-RsbZlVis6qQTlk,37526
 tuieval/scaffold.py,sha256=-AE5vKXeTtMZJ8OMk4UiIstoUWozTKjThKxZ87j0f9E,4720
 tuieval/selftest.py,sha256=xKfR73wMTYcDRSCBYQTNyKeR9bYU5HIYUSg7oZvNCyI,9330
 tuieval/tui.py,sha256=8diKtYE31ZUKq62rfMGdMkEbHgK6arRYpjFu-wcMVZs,131997
-tuieval/tune.py,sha256=zOs1cfKS3NhaFenjCY6fDb7oa8cx5lsoxgGaIEas_ys,32625
+tuieval/tune.py,sha256=Kf5angorAFNPLqeIuNdcFphMCL53BffYqjPi_XPW77Q,32879
 tuieval/verdict.py,sha256=oLKpRIruk7W8eF1YdmITFx9BgBK3kPKYViKL20Nzyu0,16306
 tuieval/watch_proxy.py,sha256=z916NRhEur04a9RnLm6ZpIEY1SnnUGPiHYbmlD9ktQ0,11035
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
@@ -42,8 +42,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev2.dist-info/METADATA,sha256=ukHZagdns1Pj_mnQQsl9dxLIl40A-E8BtYIPx57I-cw,12079
-tuieval-0.2.0.dev2.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev2.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev2.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev2.dist-info/RECORD,,
+tuieval-0.2.0.dev4.dist-info/METADATA,sha256=c653KXH4TRW-6b59HkXhbb0HW9jKyabSHWoD30aLI3w,12079
+tuieval-0.2.0.dev4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev4.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev4.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev4.dist-info/RECORD,,