tuieval 0.2.0.dev4__py3-none-any.whl → 0.2.0.dev6__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev4'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev4')
+__version__ = version = '0.2.0.dev6'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev6')
 
 __commit_id__ = commit_id = None

tuieval/run_evals.py

@@ -530,6 +530,8 @@ def cmd_tune(argv):
             say(f"{label}: started from {ws['from']}'s flags; re-tried {', '.join(ws['retested']) or 'nothing'}")
         if not pr["meta"].get("answer_guard", True):
             say(f"{label}: its answers depend on these settings; rerun its evals on this machine", "\033[33m")
+        for w in pr["meta"].get("warnings", []):
+            say(f"{label}: warning: {w}", YELLOW)
         if pr["meta"].get("rejected"):
             say(f"{label}: rejected because answers changed: {'; '.join(pr['meta']['rejected'])}")
         if a.export_pi and not export_pi(e, label):

tuieval/tui.py

@@ -2565,6 +2565,8 @@ class EvalsApp(App):
                             p = tune.tune(e, label, lambda kind, **d: on_event(kind, label=label, **d))
                             ms = p["measured"]
                             gain = f" ({tune.gain_text(ms)})" if tune.gain_text(ms) else ""
+                            if p["meta"].get("warnings"):
+                                gain += " (warning: it pushes other apps' memory to swap; see the log)"
                             if not p["meta"].get("answer_guard", True):
                                 gain += " (answers depend on these settings: rerun its evals here)"
                             done.append(f"{label}{gain}")

tuieval/tune.py

@@ -26,7 +26,9 @@ An output guard compares greedy answers with the default flags; an option that c
 noise is rejected, because speed flags must not change answers. Servers whose answers depend on
 the machine's memory settings (`outputs_depend_on_machine`) skip the guard; their tuned flags
 become part of the results fingerprint instead, so retuning marks that machine's results
-outdated. Any candidate that makes macOS swap is rejected. Servers without tune knobs are only
+outdated. A candidate under which macOS swaps more than with the defaults is rejected (it costs
+memory). Swapping the defaults already cause (the model, its context, other apps) is a warning, and
+swapping while the model loads is only noted. Servers without tune knobs are only
 measured.
 
 The knobs and their options come from models.toml [servers.<name>.tune]; placeholders {p},
@@ -48,7 +50,7 @@ from . import profiles
 
 GEN_TOKENS = 128          # generated per workload prompt
 DECODE_TOKENS = 256      # generated per prompt for the decode objective
-SWAP_LIMIT = 256 * 2**20  # a candidate that swaps out more than this is rejected
+SWAP_LIMIT = 256 * 2**20  # a candidate that swaps this much more than the defaults is rejected
 GPU_MARGIN_GB = 0.75      # a candidate whose GPU allocation comes this close to the residency limit
                           # is rejected even before it stalls: servers grow as requests arrive
 REQUEST_LIMIT_S = 600     # a tuning request taking longer fails the candidate ([tune] request_timeout_s)
@@ -69,6 +71,7 @@ class Measure:
     load_s: float | None = None
     error: str = ""
     facts: dict = dataclasses.field(default_factory=dict)   # from the server log, e.g. expert capacity
+    swapped_mb: float | None = None  # macOS swap-outs while the loaded server worked
 
     def score(self, objective):
         """Lower is better."""
@@ -461,7 +464,8 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
     knob_opts = knobs(eng, m, sv) if server.get("cmd") else {}
     fixed = list(server.get("perf", [])) if server.get("cmd") else []
     cache, starts, notes, rejected, bad = {}, [0], [], [], set()   # bad: (knob, option) that failed
-    warned = []
+    warned, warnings = [], []
+    swap_ref = [None]   # MB the defaults swapped while serving: the machine's baseline
 
     def evaluate(choices, why):
         k = tuple(sorted(choices.items()))
@@ -474,12 +478,25 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         emit("tune_step", message=f"[{starts[0]}] {why}: {' '.join(args) or '(server defaults)'}",
              start=starts[0], max_starts=max_starts)
         swap0 = machines.swapped_out_bytes()
+        swapped = None   # MB swapped out while the loaded server worked (loading itself doesn't count)
         try:
             with eng.serve(m, perf_args=fixed + args, log_name=f"{label}.tune") as (url, info):
+                # Loading may push other apps to swap once (e.g. a model locked in RAM with mlock);
+                # that's recorded, not held against the settings. Swapping while serving is.
+                loaded = machines.swapped_out_bytes()
                 r = measure(eng, m, url, work, gen_tokens, passes, emit,
                             settings.get("request_timeout_s", REQUEST_LIMIT_S))
+                end = machines.swapped_out_bytes()
+                if None not in (swap0, loaded, end):
+                    swapped = (end - loaded) / 2**20
+                    at_load = (loaded - swap0) / 2**20
+                    if at_load > SWAP_LIMIT / 2**20:
+                        emit("tune_step", message=f"    note: loading pushed {at_load:.0f} MB of other apps to swap "
+                                                  "(close apps for more headroom)")
                 r.load_s = round(info["load_s"], 1) if info["load_s"] else None
                 r.facts = dict(info["facts"])
+                if swapped is not None and at_load > SWAP_LIMIT / 2**20:
+                    r.facts["load_swapped_mb"] = round(at_load)
                 # Settled allocation after the timed pass (brief peaks while processing a prompt are
                 # harmless; sustained allocation over the limit is what makes the driver churn).
                 settled = machines.gpu_allocated_gb() if server.get("stall_guard") else None
@@ -497,18 +514,30 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                                               "close other apps for faster and fairer results")
         except engine_mod.ModelFailed as e:
             r = Measure(error=str(e).splitlines()[0])
-        swap1 = machines.swapped_out_bytes()
-        if not r.error and swap0 is not None and swap1 is not None and swap1 - swap0 > SWAP_LIMIT:
-            r.error = f"macOS swapped {(swap1 - swap0) / 2**20:.0f} MB (memory too tight)"
+        r.swapped_mb = swapped
+        # Swapping every candidate shares (model, context, prompt cache, other apps) says nothing about
+        # a flag; only swapping beyond the defaults' does.
+        extra = None if swapped is None or swap_ref[0] is None else swapped - swap_ref[0]
+        if not r.error and extra is not None and extra > SWAP_LIMIT / 2**20:
+            r.error = f"macOS swapped {extra:.0f} MB more than with the defaults (uses too much memory)"
         cache[k] = r
         emit("tune_result", message=f"    failed: {r.error}" if r.error else "    " + r.summary(), result=r, args=args)
         return r
 
+    def set_baseline(r):
+        """The defaults' swapping is the machine's baseline: warned about, never held against a flag."""
+        swap_ref[0] = r.swapped_mb or 0
+        if swap_ref[0] > SWAP_LIMIT / 2**20:
+            warnings.append(f"serving this model pushed {swap_ref[0]:.0f} MB of other apps' memory to swap on "
+                            "this machine; close other apps or lower this model's context for more headroom")
+            emit("tune_step", message="    warning: " + warnings[-1])
+
     base = None
     if not server.get("cmd") or not knob_opts:
         r = evaluate({}, "measuring (no speed knobs to tune)")
         if r.error:
             raise engine_mod.ModelFailed(r.error)
+        set_baseline(r)
         best, best_r = {}, r
         method, bench_info = "measured", None
     else:
@@ -517,6 +546,7 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         if base is None or base.error:
             raise engine_mod.ModelFailed(f"the server doesn't start with the default flags: "
                                          f"{base.error if base else 'no starts left'}")
+        set_baseline(base)
         settled, bench_info, warm_info = {}, None, None
         best, best_r, search = dict(default), base, list(knob_opts)
         sib = find_sibling(eng, m, knob_opts, sv.machine.id) \
@@ -601,7 +631,7 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                  "workload": f"{len(work)} prompts x {gen_tokens} tokens" + (f", {passes} passes" if passes > 1 else ""),
                  "llama_bench": bool(bench_info), "answer_guard": guard,
                  **({"warm_start": warm_info} if knob_opts and warm_info else {}),
-                 "rejected": rejected, "notes": notes[:10]},
+                 "rejected": rejected, "notes": notes[:10], "warnings": warnings},
     }
     path = profiles.save(sv.machine.id, label, profile, eng.tuning_dir)
     eng._serving.clear()

{tuieval-0.2.0.dev4.dist-info → tuieval-0.2.0.dev6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev4
+Version: 0.2.0.dev6
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev4.dist-info → tuieval-0.2.0.dev6.dist-info}/RECORD

@@ -1,6 +1,6 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=7rQfE9HWjmke7Ju8Nk_aqQp4chEDuGAHhvH3RPUbSyc,533
+tuieval/_version.py,sha256=9gdVeCkadGCQ8txcnOoyULu4u3PP7cNd2WZB7FPQHh8,533
 tuieval/cli.py,sha256=ZaLoWrxyCABu4SJzx5U2gVUUmwcwRgpZVqdqnxyFMOk,4167
 tuieval/client.py,sha256=Vui7ucVv_GiKObGoVS_EWSZFUKPou9PWmutqMTrahXk,9313
 tuieval/compare.py,sha256=b-rVcMwq0Tos3oNKcpUP1HXULKhAQNAS4rlXYSY7t6k,25460
@@ -10,11 +10,11 @@ tuieval/machines.py,sha256=CeestY-_6h8ZKhJbBA_xyf0prqBvjvdgs2XJtCJYnNg,12974
 tuieval/packs.py,sha256=Go-kTSPvYZriXMo4DvmqNNL-iD0J92MKLiNCnP2chJ8,10305
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=uPzidbRPUduBwP8sKyzg-U8cka0p-RsbZlVis6qQTlk,37526
+tuieval/run_evals.py,sha256=0dtPqhXxFqRLj_11I7hs2IlNkrxxilm6GOlEPfef3Fw,37625
 tuieval/scaffold.py,sha256=-AE5vKXeTtMZJ8OMk4UiIstoUWozTKjThKxZ87j0f9E,4720
 tuieval/selftest.py,sha256=xKfR73wMTYcDRSCBYQTNyKeR9bYU5HIYUSg7oZvNCyI,9330
-tuieval/tui.py,sha256=8diKtYE31ZUKq62rfMGdMkEbHgK6arRYpjFu-wcMVZs,131997
-tuieval/tune.py,sha256=Kf5angorAFNPLqeIuNdcFphMCL53BffYqjPi_XPW77Q,32879
+tuieval/tui.py,sha256=52hK8VSOVA0iHW8AyVFEyEXB32rE3U_hezJEQ5wc0TI,132159
+tuieval/tune.py,sha256=3vUll2EQkPIqRuZpAWV2-r-sk7FMPUkGeWmo63eTmHA,35031
 tuieval/verdict.py,sha256=oLKpRIruk7W8eF1YdmITFx9BgBK3kPKYViKL20Nzyu0,16306
 tuieval/watch_proxy.py,sha256=z916NRhEur04a9RnLm6ZpIEY1SnnUGPiHYbmlD9ktQ0,11035
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
@@ -42,8 +42,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev4.dist-info/METADATA,sha256=c653KXH4TRW-6b59HkXhbb0HW9jKyabSHWoD30aLI3w,12079
-tuieval-0.2.0.dev4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev4.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev4.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev4.dist-info/RECORD,,
+tuieval-0.2.0.dev6.dist-info/METADATA,sha256=3jzFS5nE1qalIrAnoAi5TBvAEZ9cyyVQoRGxbS3kvAs,12079
+tuieval-0.2.0.dev6.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev6.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev6.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev6.dist-info/RECORD,,