tuieval 0.2.0.dev5__tar.gz → 0.2.0.dev6__tar.gz

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## Unreleased
 
-- `tuieval tune` no longer rejects settings because the model's loading pushed other apps to swap (e.g. a large model locked in RAM with `--load-mode mlock`); only swapping while the loaded server works counts. Swap at load is noted in the output and the profile (`server_load_swapped_mb`).
+- `tuieval tune`: swapping the defaults already cause is a warning, not a failure; an option is rejected only if it swaps more than the defaults.
 - Warm-start tuning recognises fine-tunes whose GGUF describes the MTP draft layer differently (e.g. listing KV heads per layer) as the same model shape, so they start from an already-tuned sibling instead of tuning in full.
 - The fit check (context sized from the GGUF header, llama.cpp's memory use) only applies to servers whose command takes `{ctx}`. Servers that size their own memory keep the model's `max_context` instead of an estimate that didn't apply to them; `fit_check = true|false` on a server overrides it.
 - `tuieval export pi` has no default presets path any more: set `[export.pi] presets` to your llama.cpp router's `--models-preset` file.

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev5
+Version: 0.2.0.dev6
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/docs/models.md

@@ -78,7 +78,7 @@ The best server flags differ per model and per machine, so tuieval splits them b
 
 1. If `llama-bench` is installed, it sweeps threads, micro-batch and flash attention first (fast, no server starts).
 2. Then it starts the real server with one knob changed at a time and times a fixed **built-in** workload (three short prompts, three medium ones and one ~8k-token prompt), so tuning needs no packs and speeds are comparable between workspaces.
-3. An **output guard** rejects any option that changes greedy answers beyond noise. Candidates under which macOS swaps while the server works are rejected; swapping while the model loads (e.g. other apps making room for a model locked in RAM) is only noted.
+3. An **output guard** rejects any option that changes greedy answers beyond noise. An option under which macOS swaps more than with the defaults is rejected (it costs memory). If the defaults already push other apps to swap, the tune warns and carries on.
 
 Expect 8–15 server starts, about 20–30 minutes for a 27B model, once per model per machine. The result is saved in `tuning/<machine>/<model>.toml` and used by every later run there. Models without a profile run with each knob's first option and show *untuned*. A profile is marked for retuning when the model file or server version changes.
 

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev5'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev5')
+__version__ = version = '0.2.0.dev6'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev6')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/src/tuieval/run_evals.py

@@ -530,6 +530,8 @@ def cmd_tune(argv):
             say(f"{label}: started from {ws['from']}'s flags; re-tried {', '.join(ws['retested']) or 'nothing'}")
         if not pr["meta"].get("answer_guard", True):
             say(f"{label}: its answers depend on these settings; rerun its evals on this machine", "\033[33m")
+        for w in pr["meta"].get("warnings", []):
+            say(f"{label}: warning: {w}", YELLOW)
         if pr["meta"].get("rejected"):
             say(f"{label}: rejected because answers changed: {'; '.join(pr['meta']['rejected'])}")
         if a.export_pi and not export_pi(e, label):

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/src/tuieval/tui.py

@@ -2565,6 +2565,8 @@ class EvalsApp(App):
                             p = tune.tune(e, label, lambda kind, **d: on_event(kind, label=label, **d))
                             ms = p["measured"]
                             gain = f" ({tune.gain_text(ms)})" if tune.gain_text(ms) else ""
+                            if p["meta"].get("warnings"):
+                                gain += " (warning: it pushes other apps' memory to swap; see the log)"
                             if not p["meta"].get("answer_guard", True):
                                 gain += " (answers depend on these settings: rerun its evals here)"
                             done.append(f"{label}{gain}")

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/src/tuieval/tune.py

@@ -26,8 +26,9 @@ An output guard compares greedy answers with the default flags; an option that c
 noise is rejected, because speed flags must not change answers. Servers whose answers depend on
 the machine's memory settings (`outputs_depend_on_machine`) skip the guard; their tuned flags
 become part of the results fingerprint instead, so retuning marks that machine's results
-outdated. Any candidate under which macOS swaps while the server works is rejected (swapping
-while the model loads, e.g. to lock it in RAM, is only noted). Servers without tune knobs are only
+outdated. A candidate under which macOS swaps more than with the defaults is rejected (it costs
+memory). Swapping the defaults already cause (the model, its context, other apps) is a warning, and
+swapping while the model loads is only noted. Servers without tune knobs are only
 measured.
 
 The knobs and their options come from models.toml [servers.<name>.tune]; placeholders {p},
@@ -49,7 +50,7 @@ from . import profiles
 
 GEN_TOKENS = 128          # generated per workload prompt
 DECODE_TOKENS = 256      # generated per prompt for the decode objective
-SWAP_LIMIT = 256 * 2**20  # a candidate that swaps out more than this is rejected
+SWAP_LIMIT = 256 * 2**20  # a candidate that swaps this much more than the defaults is rejected
 GPU_MARGIN_GB = 0.75      # a candidate whose GPU allocation comes this close to the residency limit
                           # is rejected even before it stalls: servers grow as requests arrive
 REQUEST_LIMIT_S = 600     # a tuning request taking longer fails the candidate ([tune] request_timeout_s)
@@ -70,6 +71,7 @@ class Measure:
     load_s: float | None = None
     error: str = ""
     facts: dict = dataclasses.field(default_factory=dict)   # from the server log, e.g. expert capacity
+    swapped_mb: float | None = None  # macOS swap-outs while the loaded server worked
 
     def score(self, objective):
         """Lower is better."""
@@ -462,7 +464,8 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
     knob_opts = knobs(eng, m, sv) if server.get("cmd") else {}
     fixed = list(server.get("perf", [])) if server.get("cmd") else []
     cache, starts, notes, rejected, bad = {}, [0], [], [], set()   # bad: (knob, option) that failed
-    warned = []
+    warned, warnings = [], []
+    swap_ref = [None]   # MB the defaults swapped while serving: the machine's baseline
 
     def evaluate(choices, why):
         k = tuple(sorted(choices.items()))
@@ -511,17 +514,30 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                                               "close other apps for faster and fairer results")
         except engine_mod.ModelFailed as e:
             r = Measure(error=str(e).splitlines()[0])
-        if not r.error and swapped is not None and swapped > SWAP_LIMIT / 2**20:
-            r.error = f"macOS swapped {swapped:.0f} MB while the server worked (memory too tight)"
+        r.swapped_mb = swapped
+        # Swapping every candidate shares (model, context, prompt cache, other apps) says nothing about
+        # a flag; only swapping beyond the defaults' does.
+        extra = None if swapped is None or swap_ref[0] is None else swapped - swap_ref[0]
+        if not r.error and extra is not None and extra > SWAP_LIMIT / 2**20:
+            r.error = f"macOS swapped {extra:.0f} MB more than with the defaults (uses too much memory)"
         cache[k] = r
         emit("tune_result", message=f"    failed: {r.error}" if r.error else "    " + r.summary(), result=r, args=args)
         return r
 
+    def set_baseline(r):
+        """The defaults' swapping is the machine's baseline: warned about, never held against a flag."""
+        swap_ref[0] = r.swapped_mb or 0
+        if swap_ref[0] > SWAP_LIMIT / 2**20:
+            warnings.append(f"serving this model pushed {swap_ref[0]:.0f} MB of other apps' memory to swap on "
+                            "this machine; close other apps or lower this model's context for more headroom")
+            emit("tune_step", message="    warning: " + warnings[-1])
+
     base = None
     if not server.get("cmd") or not knob_opts:
         r = evaluate({}, "measuring (no speed knobs to tune)")
         if r.error:
             raise engine_mod.ModelFailed(r.error)
+        set_baseline(r)
         best, best_r = {}, r
         method, bench_info = "measured", None
     else:
@@ -530,6 +546,7 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         if base is None or base.error:
             raise engine_mod.ModelFailed(f"the server doesn't start with the default flags: "
                                          f"{base.error if base else 'no starts left'}")
+        set_baseline(base)
         settled, bench_info, warm_info = {}, None, None
         best, best_r, search = dict(default), base, list(knob_opts)
         sib = find_sibling(eng, m, knob_opts, sv.machine.id) \
@@ -614,7 +631,7 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                  "workload": f"{len(work)} prompts x {gen_tokens} tokens" + (f", {passes} passes" if passes > 1 else ""),
                  "llama_bench": bool(bench_info), "answer_guard": guard,
                  **({"warm_start": warm_info} if knob_opts and warm_info else {}),
-                 "rejected": rejected, "notes": notes[:10]},
+                 "rejected": rejected, "notes": notes[:10], "warnings": warnings},
     }
     path = profiles.save(sv.machine.id, label, profile, eng.tuning_dir)
     eng._serving.clear()

{tuieval-0.2.0.dev5 → tuieval-0.2.0.dev6}/tests/test_tuieval.py

@@ -325,14 +325,15 @@ class WarmTune(unittest.TestCase):
         self.tmp.cleanup()
 
     def _swap(self, at_load_mb, while_serving_mb):
-        """Fake macOS swap counter: each server start swaps at_load_mb while loading (between the
-        reading before the start and the one once it serves) and while_serving_mb during the work."""
+        """Fake macOS swap counter: each server start swaps at_load_mb while loading and
+        while_serving_mb(args) during the work (args: that start's flags)."""
         from tuieval import machines
         state = {"n": 0, "total": 0}
 
         def swapped_out_bytes():
             step = state["n"] % 3          # 0: before the start, 1: loaded, 2: after the work
-            state["total"] += {0: 0, 1: at_load_mb, 2: while_serving_mb}[step] * 2**20
+            mb = {0: 0, 1: at_load_mb, 2: while_serving_mb(self.eng.starts[-1]) if step == 2 else 0}[step]
+            state["total"] += mb * 2**20
             state["n"] += 1
             return state["total"]
         orig = machines.swapped_out_bytes
@@ -340,15 +341,23 @@ class WarmTune(unittest.TestCase):
         self.addCleanup(setattr, machines, "swapped_out_bytes", orig)
 
     def test_swapping_while_loading_is_only_noted(self):
-        self._swap(at_load_mb=900, while_serving_mb=0)
+        self._swap(at_load_mb=900, while_serving_mb=lambda args: 0)
         p = self.tune.tune(self.eng, "new", use_bench=False)
         self.assertEqual(p["measured"]["server_load_swapped_mb"], 900)
+        self.assertEqual(p["meta"]["warnings"], [])
 
-    def test_swapping_while_serving_fails_the_settings(self):
-        self._swap(at_load_mb=0, while_serving_mb=500)
-        with self.assertRaises(self.tune.engine_mod.ModelFailed) as cm:
-            self.tune.tune(self.eng, "new", use_bench=False)
-        self.assertIn("swapped 500 MB while the server worked", str(cm.exception))
+    def test_swapping_every_candidate_shares_is_a_warning(self):
+        self._swap(at_load_mb=0, while_serving_mb=lambda args: 800)
+        p = self.tune.tune(self.eng, "new", use_bench=False)
+        self.assertEqual(p["args"], ["-t", "6", "-ub", "256", "--spec-type", "draft-mtp"])   # tuned as usual
+        self.assertIn("800 MB", p["meta"]["warnings"][0])
+
+    def test_a_flag_that_swaps_more_than_the_defaults_is_rejected(self):
+        # -ub 256 is the fastest micro-batch here, but it costs 600 MB of swap the defaults don't
+        self._swap(at_load_mb=0, while_serving_mb=lambda args: 600 if "256" in args else 100)
+        p = self.tune.tune(self.eng, "new", use_bench=False, warm=False)
+        self.assertNotIn("256", p["args"])
+        self.assertTrue(any("500 MB more than with the defaults" in n for n in p["meta"]["notes"]))
 
     def test_family_ignores_the_mtp_layer(self):
         # one GGUF lists KV heads per layer, none on the MTP (last) layer; the other gives one number