tuieval 0.2.0.dev4__tar.gz → 0.2.0.dev5__tar.gz

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev5}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- `tuieval tune` no longer rejects settings because the model's loading pushed other apps to swap (e.g. a large model locked in RAM with `--load-mode mlock`); only swapping while the loaded server works counts. Swap at load is noted in the output and the profile (`server_load_swapped_mb`).
 - Warm-start tuning recognises fine-tunes whose GGUF describes the MTP draft layer differently (e.g. listing KV heads per layer) as the same model shape, so they start from an already-tuned sibling instead of tuning in full.
 - The fit check (context sized from the GGUF header, llama.cpp's memory use) only applies to servers whose command takes `{ctx}`. Servers that size their own memory keep the model's `max_context` instead of an estimate that didn't apply to them; `fit_check = true|false` on a server overrides it.
 - `tuieval export pi` has no default presets path any more: set `[export.pi] presets` to your llama.cpp router's `--models-preset` file.

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev4
+Version: 0.2.0.dev5
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev5}/docs/models.md

@@ -78,7 +78,7 @@ The best server flags differ per model and per machine, so tuieval splits them b
 
 1. If `llama-bench` is installed, it sweeps threads, micro-batch and flash attention first (fast, no server starts).
 2. Then it starts the real server with one knob changed at a time and times a fixed **built-in** workload (three short prompts, three medium ones and one ~8k-token prompt), so tuning needs no packs and speeds are comparable between workspaces.
-3. An **output guard** rejects any option that changes greedy answers beyond noise. Candidates that make macOS swap are rejected.
+3. An **output guard** rejects any option that changes greedy answers beyond noise. Candidates under which macOS swaps while the server works are rejected; swapping while the model loads (e.g. other apps making room for a model locked in RAM) is only noted.
 
 Expect 8–15 server starts, about 20–30 minutes for a 27B model, once per model per machine. The result is saved in `tuning/<machine>/<model>.toml` and used by every later run there. Models without a profile run with each knob's first option and show *untuned*. A profile is marked for retuning when the model file or server version changes.
 

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev5}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev4'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev4')
+__version__ = version = '0.2.0.dev5'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev5')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev5}/src/tuieval/tune.py

@@ -26,7 +26,8 @@ An output guard compares greedy answers with the default flags; an option that c
 noise is rejected, because speed flags must not change answers. Servers whose answers depend on
 the machine's memory settings (`outputs_depend_on_machine`) skip the guard; their tuned flags
 become part of the results fingerprint instead, so retuning marks that machine's results
-outdated. Any candidate that makes macOS swap is rejected. Servers without tune knobs are only
+outdated. Any candidate under which macOS swaps while the server works is rejected (swapping
+while the model loads, e.g. to lock it in RAM, is only noted). Servers without tune knobs are only
 measured.
 
 The knobs and their options come from models.toml [servers.<name>.tune]; placeholders {p},
@@ -474,12 +475,25 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         emit("tune_step", message=f"[{starts[0]}] {why}: {' '.join(args) or '(server defaults)'}",
              start=starts[0], max_starts=max_starts)
         swap0 = machines.swapped_out_bytes()
+        swapped = None   # MB swapped out while the loaded server worked (loading itself doesn't count)
         try:
             with eng.serve(m, perf_args=fixed + args, log_name=f"{label}.tune") as (url, info):
+                # Loading may push other apps to swap once (e.g. a model locked in RAM with mlock);
+                # that's recorded, not held against the settings. Swapping while serving is.
+                loaded = machines.swapped_out_bytes()
                 r = measure(eng, m, url, work, gen_tokens, passes, emit,
                             settings.get("request_timeout_s", REQUEST_LIMIT_S))
+                end = machines.swapped_out_bytes()
+                if None not in (swap0, loaded, end):
+                    swapped = (end - loaded) / 2**20
+                    at_load = (loaded - swap0) / 2**20
+                    if at_load > SWAP_LIMIT / 2**20:
+                        emit("tune_step", message=f"    note: loading pushed {at_load:.0f} MB of other apps to swap "
+                                                  "(close apps for more headroom)")
                 r.load_s = round(info["load_s"], 1) if info["load_s"] else None
                 r.facts = dict(info["facts"])
+                if swapped is not None and at_load > SWAP_LIMIT / 2**20:
+                    r.facts["load_swapped_mb"] = round(at_load)
                 # Settled allocation after the timed pass (brief peaks while processing a prompt are
                 # harmless; sustained allocation over the limit is what makes the driver churn).
                 settled = machines.gpu_allocated_gb() if server.get("stall_guard") else None
@@ -497,9 +511,8 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                                               "close other apps for faster and fairer results")
         except engine_mod.ModelFailed as e:
             r = Measure(error=str(e).splitlines()[0])
-        swap1 = machines.swapped_out_bytes()
-        if not r.error and swap0 is not None and swap1 is not None and swap1 - swap0 > SWAP_LIMIT:
-            r.error = f"macOS swapped {(swap1 - swap0) / 2**20:.0f} MB (memory too tight)"
+        if not r.error and swapped is not None and swapped > SWAP_LIMIT / 2**20:
+            r.error = f"macOS swapped {swapped:.0f} MB while the server worked (memory too tight)"
         cache[k] = r
         emit("tune_result", message=f"    failed: {r.error}" if r.error else "    " + r.summary(), result=r, args=args)
         return r

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev5}/tests/test_tuieval.py

@@ -324,6 +324,32 @@ class WarmTune(unittest.TestCase):
         self.tune.measure = self.orig
         self.tmp.cleanup()
 
+    def _swap(self, at_load_mb, while_serving_mb):
+        """Fake macOS swap counter: each server start swaps at_load_mb while loading (between the
+        reading before the start and the one once it serves) and while_serving_mb during the work."""
+        from tuieval import machines
+        state = {"n": 0, "total": 0}
+
+        def swapped_out_bytes():
+            step = state["n"] % 3          # 0: before the start, 1: loaded, 2: after the work
+            state["total"] += {0: 0, 1: at_load_mb, 2: while_serving_mb}[step] * 2**20
+            state["n"] += 1
+            return state["total"]
+        orig = machines.swapped_out_bytes
+        machines.swapped_out_bytes = swapped_out_bytes
+        self.addCleanup(setattr, machines, "swapped_out_bytes", orig)
+
+    def test_swapping_while_loading_is_only_noted(self):
+        self._swap(at_load_mb=900, while_serving_mb=0)
+        p = self.tune.tune(self.eng, "new", use_bench=False)
+        self.assertEqual(p["measured"]["server_load_swapped_mb"], 900)
+
+    def test_swapping_while_serving_fails_the_settings(self):
+        self._swap(at_load_mb=0, while_serving_mb=500)
+        with self.assertRaises(self.tune.engine_mod.ModelFailed) as cm:
+            self.tune.tune(self.eng, "new", use_bench=False)
+        self.assertIn("swapped 500 MB while the server worked", str(cm.exception))
+
     def test_family_ignores_the_mtp_layer(self):
         # one GGUF lists KV heads per layer, none on the MTP (last) layer; the other gives one number
         d = self.tmp.name