tuieval 0.2.0.dev6__py3-none-any.whl → 0.2.0.dev9__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev6'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev6')
+__version__ = version = '0.2.0.dev9'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev9')
 
 __commit_id__ = commit_id = None

tuieval/machines.py

@@ -225,6 +225,17 @@ def swapped_out_bytes():
 RESIDENCY_FRACTION = 0.5
 
 
+def memory_pressure_level():
+    """macOS memory pressure: 1 normal, 2 warning, 4 critical (the system starts ending processes).
+    None elsewhere."""
+    try:
+        out = subprocess.run(["sysctl", "-n", "kern.memorystatus_vm_pressure_level"], capture_output=True,
+                             text=True, timeout=5).stdout
+        return int(out.strip())
+    except (OSError, subprocess.TimeoutExpired, ValueError):
+        return None
+
+
 def gpu_residency_gb(machine, override=None):
     """GPU memory the driver keeps resident without churning (models.toml
     [machines.<id>] gpu_residency_gb overrides the half-of-RAM default)."""

tuieval/run_evals.py

@@ -523,7 +523,9 @@ def cmd_tune(argv):
         pr = result["profile"]
         ms = pr["measured"]
         gain = ", " + tune.gain_text(ms) if tune.gain_text(ms) else ""
-        result = f"{ms['tg_tps']} tok/s decode" if ms.get("objective") == "decode" else f"{ms['total_s']}s for the workload"
+        result = f"{ms['tg_tps']} tok/s decode" if ms.get("objective") == "decode" else \
+            f"{ms['projected_s']}s projected for the workload" if ms.get("objective") == "projected" and ms.get("projected_s") \
+            else f"{ms['total_s']}s for the workload"
         say(f"{label}: {' '.join(pr['args']) or '(no knobs)'}  ->  {result}{gain}", GREEN)
         if pr["meta"].get("warm_start"):
             ws = pr["meta"]["warm_start"]

tuieval/templates/models.toml

@@ -101,6 +101,8 @@ headers = { "X-Title" = "tuieval" }
 # request_timeout_s = 600      # a tuning request that takes longer fails that candidate
 # warm_start = true            # start from a tuned model with the same architecture and shapes (a fine-tune)
 # warm_retest = ["spec", "ubatch"]   # the knobs a warm start re-tries; the rest are inherited
+# answer_tokens = 1024         # answer length options are scored for (projected objective)
+# swap_limit_mb = 500          # reject options that swap more than this beyond the defaults (default: report only)
 
 # Optional: llama-bench for the tuner's fast first stage (found on PATH as llama-bench otherwise).
 # [bench]

tuieval/tune.py

@@ -18,18 +18,24 @@ retrained or dropped its MTP layers, and its quant mix shifts the best micro-bat
 it tunes in full instead. `tuieval tune --cold` (or [tune] warm_start = false) always tunes in full.
 
 One objective per server (models.toml `tune_objective`):
-  total   (default) total seconds for the workload: short prompts, medium ones and one long
-          (~8k-token) prompt, fixed generation length
-  decode  generated tokens per second, measured on a second pass over a few prompts after a
-          warm-up pass (for servers whose decode speed grows as their caches warm)
+  projected  (default) seconds the workload would take with full-length answers: each prompt's
+             measured reading time plus [tune] answer_tokens (default 1024) at its measured
+             generation speed. The workload generates 128 tokens per prompt, but eval answers run
+             to thousands, so generation speed counts as much as it does in real runs.
+  total      wall-clock seconds for the workload as run (128-token answers)
+  decode     generated tokens per second, measured on a second pass over a few prompts after a
+             warm-up pass (for servers whose decode speed grows as their caches warm)
 An output guard compares greedy answers with the default flags; an option that changes them beyond
 noise is rejected, because speed flags must not change answers. Servers whose answers depend on
 the machine's memory settings (`outputs_depend_on_machine`) skip the guard; their tuned flags
 become part of the results fingerprint instead, so retuning marks that machine's results
-outdated. A candidate under which macOS swaps more than with the defaults is rejected (it costs
-memory). Swapping the defaults already cause (the model, its context, other apps) is a warning, and
-swapping while the model loads is only noted. Servers without tune knobs are only
-measured.
+outdated.
+
+Memory: a candidate is rejected if macOS memory pressure turns critical while it works, or, with
+[tune] swap_limit_mb set, if it swaps more than that beyond the defaults. Otherwise memory is a
+reported cost, not a reason to reject: a flag that pushes other apps' idle memory to swap can still
+win, and the result warns about it. Swapping the defaults already cause is a warning too, and
+swapping while the model loads is only noted. Servers without tune knobs are only measured.
 
 The knobs and their options come from models.toml [servers.<name>.tune]; placeholders {p},
 {p_minus_2} and {all} are this machine's core counts.
@@ -50,7 +56,10 @@ from . import profiles
 
 GEN_TOKENS = 128          # generated per workload prompt
 DECODE_TOKENS = 256      # generated per prompt for the decode objective
-SWAP_LIMIT = 256 * 2**20  # a candidate that swaps this much more than the defaults is rejected
+SWAP_NOTABLE = 256 * 2**20  # swap worth a warning (bytes); rejecting for swap needs [tune] swap_limit_mb
+ANSWER_TOKENS = 1024       # answer length the projected objective scores ([tune] answer_tokens)
+PRESSURE_CRITICAL = 4      # macOS memory pressure level that rejects a candidate
+PRESSURE_EVERY_S = 2
 GPU_MARGIN_GB = 0.75      # a candidate whose GPU allocation comes this close to the residency limit
                           # is rejected even before it stalls: servers grow as requests arrive
 REQUEST_LIMIT_S = 600     # a tuning request taking longer fails the candidate ([tune] request_timeout_s)
@@ -72,16 +81,21 @@ class Measure:
     error: str = ""
     facts: dict = dataclasses.field(default_factory=dict)   # from the server log, e.g. expert capacity
     swapped_mb: float | None = None  # macOS swap-outs while the loaded server worked
+    projected_s: float | None = None  # the workload with full-length answers (projected objective)
+    pressure: int | None = None      # highest macOS memory pressure level while it worked
 
     def score(self, objective):
         """Lower is better."""
         if objective == "decode":
             return 1 / self.tg_tps if self.tg_tps else float("inf")
+        if objective == "projected" and self.projected_s is not None:
+            return self.projected_s
         return self.total_s
 
     def summary(self):
         facts = "".join(f" · {k} {v}" for k, v in self.facts.items())
-        return (f"{self.total_s:.1f}s total · gen {self.tg_tps} tok/s · prompt {self.pp_tps} tok/s"
+        projected = f"{self.projected_s:.0f}s projected · " if self.projected_s is not None else ""
+        return (f"{projected}{self.total_s:.1f}s total · gen {self.tg_tps} tok/s · prompt {self.pp_tps} tok/s"
                 f" · ttft {self.ttft_s}s{facts}")
 
 
@@ -194,20 +208,21 @@ def _request(eng, base_url, body, name, emit, limit):
 
 
 def measure(eng, m, base_url, work, gen_tokens=GEN_TOKENS, passes=1, emit=lambda *a, **k: None,
-            limit=REQUEST_LIMIT_S):
+            limit=REQUEST_LIMIT_S, answer_tokens=ANSWER_TOKENS):
     """Time the workload (after a warm-up request). With passes > 1 the earlier passes warm the
-    server's caches and only the last pass is reported."""
+    server's caches and only the last pass is reported. answer_tokens: the answer length the
+    projected time is for."""
     out = Measure()
     for p in range(passes):
         if passes > 1:
             emit("tune_progress", message=f"    pass {p + 1} of {passes}" + (" (warm-up)" if p < passes - 1 else " (timed)"))
-        out = _measure_once(eng, m, base_url, work, gen_tokens, emit, limit)
+        out = _measure_once(eng, m, base_url, work, gen_tokens, emit, limit, answer_tokens)
         if out.error:
             break
     return out
 
 
-def _measure_once(eng, m, base_url, work, gen_tokens, emit, limit):
+def _measure_once(eng, m, base_url, work, gen_tokens, emit, limit, answer_tokens=ANSWER_TOKENS):
     sampling = engine_mod.effective_sampling(eng.cfg, m)
     request = eng.cfg["servers"][m["server"]].get("request", {})
     timeout = eng.cfg["defaults"]["request_timeout_ms"] / 1000
@@ -218,7 +233,7 @@ def _measure_once(eng, m, base_url, work, gen_tokens, emit, limit):
     if warm["error"]:
         out.error = warm["error"]
         return out
-    ttfts, p_tok, p_s, g_tok, g_s = [], 0, 0.0, 0, 0.0
+    ttfts, p_tok, p_s, g_tok, g_s, projected = [], 0, 0.0, 0, 0.0, 0.0
     for name, msgs in work:
         eng._check()
         emit("tune_progress", message=f"    {name} ({gen_tokens} tokens)")
@@ -230,6 +245,13 @@ def _measure_once(eng, m, base_url, work, gen_tokens, emit, limit):
         emit("tune_progress", message=f"      done: {met['completion_tokens']} tokens in {res['total_s']:.1f}s"
                                       + (f", {met['gen_tps']} tok/s" if met["gen_tps"] else ""))
         out.total_s += res["total_s"]
+        # The same request with a full-length answer: its reading time plus answer_tokens at the
+        # generation speed measured here (as run if the server doesn't report that speed).
+        if met["completion_tokens"] and met["gen_tps"]:
+            reading = max(res["total_s"] - met["completion_tokens"] / met["gen_tps"], 0.0)
+            projected += reading + answer_tokens / met["gen_tps"]
+        else:
+            projected += res["total_s"]
         out.texts.append((res["reasoning"] or "") + (res["answer"] or ""))
         if name != "long" and res.get("ttft_s") is not None:
             ttfts.append(res["ttft_s"])
@@ -238,6 +260,7 @@ def _measure_once(eng, m, base_url, work, gen_tokens, emit, limit):
         if met["completion_tokens"] and met["gen_tps"]:
             g_tok, g_s = g_tok + met["completion_tokens"], g_s + met["completion_tokens"] / met["gen_tps"]
     out.total_s = round(out.total_s, 2)
+    out.projected_s = round(projected, 1)
     out.ttft_s = round(sum(ttfts) / len(ttfts), 3) if ttfts else None
     out.pp_tps = round(p_tok / p_s, 1) if p_s else None
     out.tg_tps = round(g_tok / g_s, 2) if g_s else None
@@ -246,12 +269,37 @@ def _measure_once(eng, m, base_url, work, gen_tokens, emit, limit):
 
 def gain_pct(objective, base, best):
     """How much better the chosen settings are than the defaults: % more decode tok/s for the
-    decode objective, % less total time otherwise."""
+    decode objective, % less (projected or total) time otherwise."""
     if base is None:
         return None
     if objective == "decode":
         return round(100 * (best.tg_tps / base.tg_tps - 1)) if base.tg_tps and best.tg_tps else None
-    return round(100 * (1 - best.total_s / base.total_s)) if base.total_s else None
+    b, n = base.score(objective), best.score(objective)
+    return round(100 * (1 - n / b)) if b else None
+
+
+class PressureWatch:
+    """The highest macOS memory pressure level seen while a block runs (None where unknown)."""
+
+    def __enter__(self):
+        self.level, self._stop = machines.memory_pressure_level(), threading.Event()
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._thread.start()
+        return self
+
+    def _sample(self):
+        lvl = machines.memory_pressure_level()
+        if lvl is not None:
+            self.level = max(self.level or 0, lvl)
+
+    def _run(self):
+        while not self._stop.wait(PRESSURE_EVERY_S):
+            self._sample()
+
+    def __exit__(self, *exc):
+        self._stop.set()
+        self._thread.join(PRESSURE_EVERY_S + 1)
+        self._sample()
 
 
 def gain_text(measured):
@@ -453,12 +501,14 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
     sv = eng.serving(m)
     if not sv.fits:
         raise engine_mod.ModelFailed(sv.fit_note)
-    objective = server.get("tune_objective", "total")
+    objective = server.get("tune_objective", "projected")
     if objective == "decode":
         work, gen_tokens, passes = decode_workload(eng), DECODE_TOKENS, 2
     else:
         work, gen_tokens, passes = workload(eng, sv.ctx), GEN_TOKENS, 1
     settings = eng.cfg.get("tune", {})
+    answer_tokens = settings.get("answer_tokens", ANSWER_TOKENS)
+    swap_limit = settings.get("swap_limit_mb")   # MB beyond the defaults' swap that rejects; None: report only
     threshold = settings.get("guard_similarity", 0.6)
     guard = not server.get("outputs_depend_on_machine")
     knob_opts = knobs(eng, m, sv) if server.get("cmd") else {}
@@ -484,18 +534,20 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                 # Loading may push other apps to swap once (e.g. a model locked in RAM with mlock);
                 # that's recorded, not held against the settings. Swapping while serving is.
                 loaded = machines.swapped_out_bytes()
-                r = measure(eng, m, url, work, gen_tokens, passes, emit,
-                            settings.get("request_timeout_s", REQUEST_LIMIT_S))
+                with PressureWatch() as pressure:
+                    r = measure(eng, m, url, work, gen_tokens, passes, emit,
+                                settings.get("request_timeout_s", REQUEST_LIMIT_S), answer_tokens=answer_tokens)
                 end = machines.swapped_out_bytes()
+                r.pressure = pressure.level
                 if None not in (swap0, loaded, end):
                     swapped = (end - loaded) / 2**20
                     at_load = (loaded - swap0) / 2**20
-                    if at_load > SWAP_LIMIT / 2**20:
+                    if at_load > SWAP_NOTABLE / 2**20:
                         emit("tune_step", message=f"    note: loading pushed {at_load:.0f} MB of other apps to swap "
                                                   "(close apps for more headroom)")
                 r.load_s = round(info["load_s"], 1) if info["load_s"] else None
                 r.facts = dict(info["facts"])
-                if swapped is not None and at_load > SWAP_LIMIT / 2**20:
+                if swapped is not None and at_load > SWAP_NOTABLE / 2**20:
                     r.facts["load_swapped_mb"] = round(at_load)
                 # Settled allocation after the timed pass (brief peaks while processing a prompt are
                 # harmless; sustained allocation over the limit is what makes the driver churn).
@@ -516,10 +568,15 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
             r = Measure(error=str(e).splitlines()[0])
         r.swapped_mb = swapped
         # Swapping every candidate shares (model, context, prompt cache, other apps) says nothing about
-        # a flag; only swapping beyond the defaults' does.
+        # a flag; only swapping beyond the defaults' does, and it's a cost to report unless it harms:
+        # critical memory pressure, or more than the user's swap_limit_mb.
         extra = None if swapped is None or swap_ref[0] is None else swapped - swap_ref[0]
-        if not r.error and extra is not None and extra > SWAP_LIMIT / 2**20:
-            r.error = f"macOS swapped {extra:.0f} MB more than with the defaults (uses too much memory)"
+        if extra is not None and extra > SWAP_NOTABLE / 2**20:
+            r.facts["extra_swap_mb"] = round(extra)
+        if not r.error and r.pressure is not None and r.pressure >= PRESSURE_CRITICAL:
+            r.error = "macOS memory pressure turned critical (memory too tight)"
+        elif not r.error and swap_limit is not None and extra is not None and extra > swap_limit:
+            r.error = f"macOS swapped {extra:.0f} MB more than with the defaults (over swap_limit_mb = {swap_limit})"
         cache[k] = r
         emit("tune_result", message=f"    failed: {r.error}" if r.error else "    " + r.summary(), result=r, args=args)
         return r
@@ -527,7 +584,7 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
     def set_baseline(r):
         """The defaults' swapping is the machine's baseline: warned about, never held against a flag."""
         swap_ref[0] = r.swapped_mb or 0
-        if swap_ref[0] > SWAP_LIMIT / 2**20:
+        if swap_ref[0] > SWAP_NOTABLE / 2**20:
             warnings.append(f"serving this model pushed {swap_ref[0]:.0f} MB of other apps' memory to swap on "
                             "this machine; close other apps or lower this model's context for more headroom")
             emit("tune_step", message="    warning: " + warnings[-1])
@@ -615,6 +672,11 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
         if warm_info:
             notes.insert(0, f"warm start from {warm_info['from']}: inherited "
                             f"{', '.join(warm_info['inherited']) or 'nothing'}")
+    if best_r.facts.get("extra_swap_mb") and best_r is not base:
+        warnings.append(f"the chosen flags use more memory than the defaults: macOS swapped "
+                        f"~{best_r.facts['extra_swap_mb']} MB more of other apps' memory; close apps during runs, "
+                        "or set [tune] swap_limit_mb to rule such flags out")
+        emit("tune_step", message="    warning: " + warnings[-1])
     size = sv.identity.get("model_bytes")
     profile = {
         "args": resolve(knob_opts, best),
@@ -623,12 +685,15 @@ def tune(eng, label, emit=lambda *a, **k: None, max_starts=16, min_gain=0.03, us
                      "tg_tps": best_r.tg_tps, "load_s": best_r.load_s,
                      "default_total_s": base.total_s if knob_opts else None,
                      "default_tg_tps": base.tg_tps if knob_opts else None, "objective": objective,
+                     "projected_s": best_r.projected_s,
+                     "default_projected_s": base.projected_s if knob_opts else None,
                      **{f"server_{k}": v for k, v in best_r.facts.items()},
                      "gain_pct": gain_pct(objective, base, best_r)},
         "meta": {"method": method, "date": profiles.now(), "machine": sv.machine.summary,
                  "server_version": eng.server_version(m), "model_file": os.path.basename(m["model"]),
                  "model_bytes": size, "ctx": sv.ctx, "server_starts": starts[0],
-                 "workload": f"{len(work)} prompts x {gen_tokens} tokens" + (f", {passes} passes" if passes > 1 else ""),
+                 "workload": f"{len(work)} prompts x {gen_tokens} tokens" + (f", {passes} passes" if passes > 1 else "")
+                 + (f", scored for {answer_tokens}-token answers" if objective == "projected" else ""),
                  "llama_bench": bool(bench_info), "answer_guard": guard,
                  **({"warm_start": warm_info} if knob_opts and warm_info else {}),
                  "rejected": rejected, "notes": notes[:10], "warnings": warnings},

{tuieval-0.2.0.dev6.dist-info → tuieval-0.2.0.dev9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev6
+Version: 0.2.0.dev9
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues
@@ -26,7 +26,9 @@ Description-Content-Type: text/markdown
 
 <p align="center"><img src="https://raw.githubusercontent.com/ashe-wb/tuieval/main/docs/images/brand/readme-header.png" alt="tuieval" width="100%"></p>
 
-**Find out which model you can trust with your work, on your own questions, in the terminal.** Run the same eval packs against local models (llama.cpp GGUFs, LM Studio, Ollama, vLLM; tiny, dense or MoE) and against any frontier model on OpenRouter, side by side. tuieval measures **accuracy, speed and token use** together, and gives a **PASS / FAIL / INCONCLUSIVE** verdict per use case. Grading is automatic; there's no LLM judge.
+**Find out which model you can trust with your work, on your own questions, in the terminal.**
+
+Run the same eval packs against local models (llama.cpp GGUFs, LM Studio, Ollama, vLLM; tiny, dense or MoE) and against any frontier model on OpenRouter, side by side. tuieval measures **accuracy, speed and token use** together, and gives a **PASS / FAIL / INCONCLUSIVE** verdict per use case. Grading is automatic; there's no LLM judge.
 
 Public benchmarks leak into training data and rarely look like your work, so tuieval ships with **no built-in benchmark**. Instead you build *eval packs* from what you actually do: questions with checkable answers, in your domain, with your rules. Then you get an answer to the questions that matter:
 

{tuieval-0.2.0.dev6.dist-info → tuieval-0.2.0.dev9.dist-info}/RECORD

@@ -1,20 +1,20 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=9gdVeCkadGCQ8txcnOoyULu4u3PP7cNd2WZB7FPQHh8,533
+tuieval/_version.py,sha256=IHVwmalkoW20mASroKhIhmhNf5irHisjdIrjTxJWRpc,533
 tuieval/cli.py,sha256=ZaLoWrxyCABu4SJzx5U2gVUUmwcwRgpZVqdqnxyFMOk,4167
 tuieval/client.py,sha256=Vui7ucVv_GiKObGoVS_EWSZFUKPou9PWmutqMTrahXk,9313
 tuieval/compare.py,sha256=b-rVcMwq0Tos3oNKcpUP1HXULKhAQNAS4rlXYSY7t6k,25460
 tuieval/engine.py,sha256=SizirCzV1klc_a2nS14FIjS5jLiHCGbVT23eES4StEc,94817
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
-tuieval/machines.py,sha256=CeestY-_6h8ZKhJbBA_xyf0prqBvjvdgs2XJtCJYnNg,12974
+tuieval/machines.py,sha256=ImEihNj8QlBRdCHLED4QIdjJUvjVtZgo6DiVU5h6Zzs,13415
 tuieval/packs.py,sha256=Go-kTSPvYZriXMo4DvmqNNL-iD0J92MKLiNCnP2chJ8,10305
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
-tuieval/run_evals.py,sha256=0dtPqhXxFqRLj_11I7hs2IlNkrxxilm6GOlEPfef3Fw,37625
+tuieval/run_evals.py,sha256=hi-bKDjOHQjmAvWRV9-lxJGcUSvHnP36gUGkHeSFLOc,37773
 tuieval/scaffold.py,sha256=-AE5vKXeTtMZJ8OMk4UiIstoUWozTKjThKxZ87j0f9E,4720
 tuieval/selftest.py,sha256=xKfR73wMTYcDRSCBYQTNyKeR9bYU5HIYUSg7oZvNCyI,9330
 tuieval/tui.py,sha256=52hK8VSOVA0iHW8AyVFEyEXB32rE3U_hezJEQ5wc0TI,132159
-tuieval/tune.py,sha256=3vUll2EQkPIqRuZpAWV2-r-sk7FMPUkGeWmo63eTmHA,35031
+tuieval/tune.py,sha256=jXwYuSjfdqbLCE3WonP2xaUHwk_FnhI67twB1BaHggI,38992
 tuieval/verdict.py,sha256=oLKpRIruk7W8eF1YdmITFx9BgBK3kPKYViKL20Nzyu0,16306
 tuieval/watch_proxy.py,sha256=z916NRhEur04a9RnLm6ZpIEY1SnnUGPiHYbmlD9ktQ0,11035
 tuieval/workspace.py,sha256=V_HhkaN1odr0OpjNsPYradRWLpzDldM7SwpvxohBL4U,895
@@ -25,7 +25,7 @@ tuieval/graders/code.py,sha256=mNoPw-C_lNaXTTU57TJjhcVPTvdR400PxoE9Ow-tgHg,3703
 tuieval/graders/rag.py,sha256=j_gEjmaXmY_hgKu7-7pVQpQ8Cwb49gadnwFurih-1Hw,1447
 tuieval/graders/reply.py,sha256=BdkL7EWSSMoyjwCOFa97TTQMR1QyrpMH-ntcgjzDPEg,2199
 tuieval/graders/tool_call.py,sha256=_QXURtOk4ZKBnxXVMzFL1c9jWLWksRlLZ4VqOCL0t90,3578
-tuieval/templates/models.toml,sha256=bZRpQgj2LFeEQhnqkf0YUhrGybrugCilyWTg19_W4g4,7879
+tuieval/templates/models.toml,sha256=hwXF1aFGNPyg3rQ-V29-BwVXCnQM2KoJG1RBB-l3_O8,8087
 tuieval/templates/packs/answer/pack.toml,sha256=8bjMBxG1Zz3yxEnYCqRTNBJ-JGYlquqiK19p8URWwTc,1167
 tuieval/templates/packs/answer/system.txt,sha256=m4V6X1i49Bas2HowZMWgfdDSPDHTObb_WxHXt0xw1Yo,248
 tuieval/templates/packs/answer/tests.yaml,sha256=WiYPOknPcO9E6f1wXhLcfA0W5v-oK8MgZV956uWtcUQ,2660
@@ -42,8 +42,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev6.dist-info/METADATA,sha256=3jzFS5nE1qalIrAnoAi5TBvAEZ9cyyVQoRGxbS3kvAs,12079
-tuieval-0.2.0.dev6.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev6.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev6.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev6.dist-info/RECORD,,
+tuieval-0.2.0.dev9.dist-info/METADATA,sha256=BuLF9eKFTrAyKzsz2etiwhH3qBrr1Gv_Czy0VGrHx5A,12080
+tuieval-0.2.0.dev9.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev9.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev9.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev9.dist-info/RECORD,,