tuieval 0.2.0.dev4__tar.gz → 0.2.0.dev6__tar.gz

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- `tuieval tune`: swapping the defaults already cause is a warning, not a failure; an option is rejected only if it swaps more than the defaults.
 - Warm-start tuning recognises fine-tunes whose GGUF describes the MTP draft layer differently (e.g. listing KV heads per layer) as the same model shape, so they start from an already-tuned sibling instead of tuning in full.
 - The fit check (context sized from the GGUF header, llama.cpp's memory use) only applies to servers whose command takes `{ctx}`. Servers that size their own memory keep the model's `max_context` instead of an estimate that didn't apply to them; `fit_check = true|false` on a server overrides it.
 - `tuieval export pi` has no default presets path any more: set `[export.pi] presets` to your llama.cpp router's `--models-preset` file.

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev4
+Version: 0.2.0.dev6
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/docs/models.md

@@ -78,7 +78,7 @@ The best server flags differ per model and per machine, so tuieval splits them b
 
 1. If `llama-bench` is installed, it sweeps threads, micro-batch and flash attention first (fast, no server starts).
 2. Then it starts the real server with one knob changed at a time and times a fixed **built-in** workload (three short prompts, three medium ones and one ~8k-token prompt), so tuning needs no packs and speeds are comparable between workspaces.
-3. An **output guard** rejects any option that changes greedy answers beyond noise. Candidates that make macOS swap are rejected.
+3. An **output guard** rejects any option that changes greedy answers beyond noise. An option under which macOS swaps more than with the defaults is rejected (it costs memory). If the defaults already push other apps to swap, the tune warns and carries on.
 
 Expect 8–15 server starts, about 20–30 minutes for a 27B model, once per model per machine. The result is saved in `tuning/<machine>/<model>.toml` and used by every later run there. Models without a profile run with each knob's first option and show *untuned*. A profile is marked for retuning when the model file or server version changes.
 

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev4'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev4')
+__version__ = version = '0.2.0.dev6'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev6')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/src/tuieval/run_evals.py

@@ -530,6 +530,8 @@ def cmd_tune(argv):
             say(f"{label}: started from {ws['from']}'s flags; re-tried {', '.join(ws['retested']) or 'nothing'}")
         if not pr["meta"].get("answer_guard", True):
             say(f"{label}: its answers depend on these settings; rerun its evals on this machine", "\033[33m")
+        for w in pr["meta"].get("warnings", []):
+            say(f"{label}: warning: {w}", YELLOW)
         if pr["meta"].get("rejected"):
             say(f"{label}: rejected because answers changed: {'; '.join(pr['meta']['rejected'])}")
         if a.export_pi and not export_pi(e, label):

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/src/tuieval/tui.py

@@ -2565,6 +2565,8 @@ class EvalsApp(App):
                             p = tune.tune(e, label, lambda kind, **d: on_event(kind, label=label, **d))
                             ms = p["measured"]
                             gain = f" ({tune.gain_text(ms)})" if tune.gain_text(ms) else ""
+                            if p["meta"].get("warnings"):
+                                gain += " (warning: it pushes other apps' memory to swap; see the log)"
                             if not p["meta"].get("answer_guard", True):
                                 gain += " (answers depend on these settings: rerun its evals here)"
                             done.append(f"{label}{gain}")

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/src/tuieval/tune.py

@@ -26,7 +26,9 @@ An output guard compares greedy answers with the default flags; an option that c
 noise is rejected, because speed flags must not change answers. Servers whose answers depend on
 the machine's memory settings (`outputs_depend_on_machine`) skip the guard; their tuned flags
 become part of the results fingerprint instead, so retuning marks that machine's results
-outdated. Any candidate that makes macOS swap is rejected. Servers without tune knobs are only
+outdated. A candidate under which macOS swaps more than with the defaults is rejected (it costs
+memory). Swapping the defaults already cause (the model, its context, other apps) is a warning, and
+swapping while the model loads is only noted. Servers without tune knobs are only
 measured.
 
 The knobs and their options come from models.toml [servers.<name>.tune]; placeholders {p},
@@ -48,7 +50,7 @@ from . import profiles
 
 GEN_TOKENS = 128          # generated per workload prompt
 DECODE_TOKENS = 256      # generated per prompt for the decode objective
-SWAP_LIMIT = 256 * 2**20  # a candidate that swaps out more than this is rejected
+SWAP_LIMIT = 256 * 2**20  # a candidate that swaps this much more than the defaults is rejected
 GPU_MARGIN_GB = 0.75      # a candidate whose GPU allocation comes this close to the residency limit
                           # is rejected even before it stalls: servers grow as requests arrive
 REQUEST_LIMIT_S = 600     # a tuning request taking longer fails the candidate ([tune] request_timeout_s)
@@ -69,6 +71,7 @@ class Measure:
     load_s: float | None = None
     error: str = ""
     facts: dict = dataclasses.field(default_factory=dict)   # from the server log, e.g. expert capacity
+    swapped_mb: float | None = None  # macOS swap-outs while the loaded server worked
 
     def score(self, objective):
         """Lower is better."""
@@ -461,7 +464,8 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
     knob_opts = knobs(eng, m, sv) if server.get("cmd") else {}
     fixed = list(server.get("perf", [])) if server.get("cmd") else []
     cache, starts, notes, rejected, bad = {}, [0], [], [], set()   # bad: (knob, option) that failed
-    warned = []
+    warned, warnings = [], []
+    swap_ref = [None]   # MB the defaults swapped while serving: the machine's baseline
 
     def evaluate(choices, why):
         k = tuple(sorted(choices.items()))
@@ -474,12 +478,25 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         emit("tune_step", message=f"[{starts[0]}] {why}: {' '.join(args) or '(server defaults)'}",
              start=starts[0], max_starts=max_starts)
         swap0 = machines.swapped_out_bytes()
+        swapped = None   # MB swapped out while the loaded server worked (loading itself doesn't count)
         try:
             with eng.serve(m, perf_args=fixed + args, log_name=f"{label}.tune") as (url, info):
+                # Loading may push other apps to swap once (e.g. a model locked in RAM with mlock);
+                # that's recorded, not held against the settings. Swapping while serving is.
+                loaded = machines.swapped_out_bytes()
                 r = measure(eng, m, url, work, gen_tokens, passes, emit,
                             settings.get("request_timeout_s", REQUEST_LIMIT_S))
+                end = machines.swapped_out_bytes()
+                if None not in (swap0, loaded, end):
+                    swapped = (end - loaded) / 2**20
+                    at_load = (loaded - swap0) / 2**20
+                    if at_load > SWAP_LIMIT / 2**20:
+                        emit("tune_step", message=f"    note: loading pushed {at_load:.0f} MB of other apps to swap "
+                                                  "(close apps for more headroom)")
                 r.load_s = round(info["load_s"], 1) if info["load_s"] else None
                 r.facts = dict(info["facts"])
+                if swapped is not None and at_load > SWAP_LIMIT / 2**20:
+                    r.facts["load_swapped_mb"] = round(at_load)
                 # Settled allocation after the timed pass (brief peaks while processing a prompt are
                 # harmless; sustained allocation over the limit is what makes the driver churn).
                 settled = machines.gpu_allocated_gb() if server.get("stall_guard") else None
@@ -497,18 +514,30 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                                               "close other apps for faster and fairer results")
         except engine_mod.ModelFailed as e:
             r = Measure(error=str(e).splitlines()[0])
-        swap1 = machines.swapped_out_bytes()
-        if not r.error and swap0 is not None and swap1 is not None and swap1 - swap0 > SWAP_LIMIT:
-            r.error = f"macOS swapped {(swap1 - swap0) / 2**20:.0f} MB (memory too tight)"
+        r.swapped_mb = swapped
+        # Swapping every candidate shares (model, context, prompt cache, other apps) says nothing about
+        # a flag; only swapping beyond the defaults' does.
+        extra = None if swapped is None or swap_ref[0] is None else swapped - swap_ref[0]
+        if not r.error and extra is not None and extra > SWAP_LIMIT / 2**20:
+            r.error = f"macOS swapped {extra:.0f} MB more than with the defaults (uses too much memory)"
         cache[k] = r
         emit("tune_result", message=f"    failed: {r.error}" if r.error else "    " + r.summary(), result=r, args=args)
         return r
 
+    def set_baseline(r):
+        """The defaults' swapping is the machine's baseline: warned about, never held against a flag."""
+        swap_ref[0] = r.swapped_mb or 0
+        if swap_ref[0] > SWAP_LIMIT / 2**20:
+            warnings.append(f"serving this model pushed {swap_ref[0]:.0f} MB of other apps' memory to swap on "
+                            "this machine; close other apps or lower this model's context for more headroom")
+            emit("tune_step", message="    warning: " + warnings[-1])
+
     base = None
     if not server.get("cmd") or not knob_opts:
         r = evaluate({}, "measuring (no speed knobs to tune)")
         if r.error:
             raise engine_mod.ModelFailed(r.error)
+        set_baseline(r)
         best, best_r = {}, r
         method, bench_info = "measured", None
     else:
@@ -517,6 +546,7 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         if base is None or base.error:
             raise engine_mod.ModelFailed(f"the server doesn't start with the default flags: "
                                          f"{base.error if base else 'no starts left'}")
+        set_baseline(base)
         settled, bench_info, warm_info = {}, None, None
         best, best_r, search = dict(default), base, list(knob_opts)
         sib = find_sibling(eng, m, knob_opts, sv.machine.id) \
@@ -601,7 +631,7 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                  "workload": f"{len(work)} prompts x {gen_tokens} tokens" + (f", {passes} passes" if passes > 1 else ""),
                  "llama_bench": bool(bench_info), "answer_guard": guard,
                  **({"warm_start": warm_info} if knob_opts and warm_info else {}),
-                 "rejected": rejected, "notes": notes[:10]},
+                 "rejected": rejected, "notes": notes[:10], "warnings": warnings},
     }
     path = profiles.save(sv.machine.id, label, profile, eng.tuning_dir)
     eng._serving.clear()

{tuieval-0.2.0.dev4 → tuieval-0.2.0.dev6}/tests/test_tuieval.py

@@ -324,6 +324,41 @@ class WarmTune(unittest.TestCase):
         self.tune.measure = self.orig
         self.tmp.cleanup()
 
+    def _swap(self, at_load_mb, while_serving_mb):
+        """Fake macOS swap counter: each server start swaps at_load_mb while loading and
+        while_serving_mb(args) during the work (args: that start's flags)."""
+        from tuieval import machines
+        state = {"n": 0, "total": 0}
+
+        def swapped_out_bytes():
+            step = state["n"] % 3          # 0: before the start, 1: loaded, 2: after the work
+            mb = {0: 0, 1: at_load_mb, 2: while_serving_mb(self.eng.starts[-1]) if step == 2 else 0}[step]
+            state["total"] += mb * 2**20
+            state["n"] += 1
+            return state["total"]
+        orig = machines.swapped_out_bytes
+        machines.swapped_out_bytes = swapped_out_bytes
+        self.addCleanup(setattr, machines, "swapped_out_bytes", orig)
+
+    def test_swapping_while_loading_is_only_noted(self):
+        self._swap(at_load_mb=900, while_serving_mb=lambda args: 0)
+        p = self.tune.tune(self.eng, "new", use_bench=False)
+        self.assertEqual(p["measured"]["server_load_swapped_mb"], 900)
+        self.assertEqual(p["meta"]["warnings"], [])
+
+    def test_swapping_every_candidate_shares_is_a_warning(self):
+        self._swap(at_load_mb=0, while_serving_mb=lambda args: 800)
+        p = self.tune.tune(self.eng, "new", use_bench=False)
+        self.assertEqual(p["args"], ["-t", "6", "-ub", "256", "--spec-type", "draft-mtp"])   # tuned as usual
+        self.assertIn("800 MB", p["meta"]["warnings"][0])
+
+    def test_a_flag_that_swaps_more_than_the_defaults_is_rejected(self):
+        # -ub 256 is the fastest micro-batch here, but it costs 600 MB of swap the defaults don't
+        self._swap(at_load_mb=0, while_serving_mb=lambda args: 600 if "256" in args else 100)
+        p = self.tune.tune(self.eng, "new", use_bench=False, warm=False)
+        self.assertNotIn("256", p["args"])
+        self.assertTrue(any("500 MB more than with the defaults" in n for n in p["meta"]["notes"]))
+
     def test_family_ignores_the_mtp_layer(self):
         # one GGUF lists KV heads per layer, none on the MTP (last) layer; the other gives one number
         d = self.tmp.name