tuieval 0.2.0.dev87__tar.gz → 0.2.0.dev89__tar.gz

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- `max_tokens` in pack.toml sets one pack's output budget; changing it reruns only that pack. Models whose context or endpoint can't fit it are skipped for that pack.
 - The run screen's queue keeps the running model in view, unless you scroll it yourself (scrolling back to it follows again).
 - Start while a run is going skips models and packs already running or queued (it says which); picking only those adds nothing. Retest and picked questions still go through.
 - Fix: something queued just as a run ended (while its server was stopping) now starts instead of waiting forever.

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev87
+Version: 0.2.0.dev89
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/docs/writing-packs.md

@@ -35,6 +35,7 @@ order = 30                       # position in the list
 tests = ["tests.yaml"]           # optional; default: tests.* first, then every other .yaml/.csv file
 screen = 20                      # tests in a Screen run (spread across categories, one variant per group)
 stop_early = true                # stop a run of this pack once its FAIL is certain (default false; see below)
+max_tokens = 32768               # optional: this pack's output budget, instead of models.toml [sampling] max_tokens
 
 [certify]
 repeat = 3                       # repeats in a Certify run (default: models.toml [defaults] repeat)
@@ -53,6 +54,8 @@ min_consistency = 0.95           # share of variant groups where every variant p
 
 `needs` entries other than `vision`, `tools` and `long_context` name Python modules the pack's grading needs (for example `pandas`, when your hidden tests use it). The pack is skipped, with a note, where that module isn't installed in tuieval's Python environment.
 
+`max_tokens` suits packs whose answers are much longer (or shorter) than the rest. Thinking counts toward it. It's part of the pack's fingerprint, so changing it reruns only this pack. A model is skipped for the pack, with a note, when its context can't hold the longest prompt plus `max_tokens`, or when its pinned provider endpoint allows fewer output tokens.
+
 ## Tests (YAML)
 
 ```yaml

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev87'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev87')
+__version__ = version = '0.2.0.dev89'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev89')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/src/tuieval/engine.py

@@ -646,8 +646,9 @@ def serves(ids, name):
 
 
 def pack_min_context(pack):
-    """Rough context a pack needs: its longest prompt (~4 chars/token) plus room to answer."""
-    return max(len(t["input"]) for t in pack.tests) // 4 + 4096
+    """Rough context a pack needs: its longest prompt (~4 chars/token) plus room to answer (the
+    pack's own max_tokens when it sets one)."""
+    return max(len(t["input"]) for t in pack.tests) // 4 + (pack.max_tokens or 4096)
 
 
 class Cancelled(Exception):
@@ -872,7 +873,8 @@ class Engine:
                 needs_tools = m.get("tools") is not False and any("tools" in p.needs for p in self.packs.values())
                 ep = choose_endpoint(eps, m["model"], effective_sampling(self.cfg, m), needs_tools, previous)
                 ep = ep and {"tag": ep["tag"], "provider": ep.get("provider_name"),
-                             "quantization": ep.get("quantization") or "unknown"}
+                             "quantization": ep.get("quantization") or "unknown",
+                             "max_output": ep.get("max_completion_tokens")}
             except (OSError, ValueError):
                 # offline: assume the endpoint earlier results used (the run itself will need the network)
                 ep = {"tag": previous, "provider": None, "quantization": previous_quant} if previous else None
@@ -1196,6 +1198,9 @@ class Engine:
                 elif (sv.ctx or 10**9) < pack_min_context(pack):
                     job.status, job.note = "skipped", \
                         f"needs ~{pack_min_context(pack) // 1024}k context, {sv.ctx // 1024}k fits on {sv.machine.id}"
+                elif pack.max_tokens and pack.max_tokens > ((self.endpoint(m) or {}).get("max_output") or 10**9):
+                    job.status, job.note = "skipped", \
+                        f"needs {pack.max_tokens:,} output tokens, endpoint allows {self.endpoint(m)['max_output']:,}"
                 elif missing:
                     job.status, job.note = "skipped", f"{', '.join(missing)} not installed in tuieval's Python"
                 elif status and status.startswith("outdated"):
@@ -2037,7 +2042,8 @@ class Engine:
     def _body(self, job, test, rep=0):
         s = job.sampling
         body = {"model": job.model["served_name"], "messages": self._messages(job, test),
-                "temperature": s["temperature"], "top_p": s["top_p"], "max_tokens": s["max_tokens"]}
+                "temperature": s["temperature"], "top_p": s["top_p"],
+                "max_tokens": job.pack.max_tokens or s["max_tokens"]}
         for k in ("top_k", "min_p", "presence_penalty", "repeat_penalty", "seed", "reasoning_effort"):
             if k in s:
                 body[k] = s[k]
@@ -2380,7 +2386,8 @@ class Engine:
             "model": {**{k: m.get(k) for k in ("label", "served_name", "server", "model", "tags", "vision", "mmproj")},
                       **{k: m[k] for k in ("tools", "max_context", "thinking") if k in m}},
             "pack": {"name": job.pack.name, "label": job.pack.label, "group": job.pack.group,
-                     "fingerprint": job.pack.fingerprint, "tests": len(job.pack.tests)},
+                     "fingerprint": job.pack.fingerprint, "tests": len(job.pack.tests),
+                     **({"max_tokens": job.pack.max_tokens} if job.pack.max_tokens else {})},
             "settings": job.sampling,
             "serving": job.settings.get("serving"),
             "settings_fingerprint": settings_fingerprint(job.settings),

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/src/tuieval/packs.py

@@ -50,6 +50,7 @@ class Pack:
     screen: int            # tests in a screening run
     certify_repeat: int    # repeats in a certification run (0 = models.toml default)
     stop_early: bool = False   # stop the pack once its FAIL is certain (pack.toml stop_early)
+    max_tokens: int = 0    # output budget for this pack's answers (0 = [sampling] max_tokens)
 
     @property
     def modules(self):
@@ -135,12 +136,15 @@ NOT_FINGERPRINTED = {"reference", "wrong", "description", "category", "critical"
 DIFFICULTIES = ("easy", "medium", "hard")
 
 
-def _fingerprint(system, tools, grader, tests, images):
-    """Hash of what the model sees and what decides pass/fail."""
+def _fingerprint(system, tools, grader, tests, images, max_tokens=0):
+    """Hash of what the model sees and what decides pass/fail. A pack's own max_tokens counts
+    (it decides what gets cut off); packs without one keep the fingerprint they always had."""
     h = hashlib.sha256()
-    h.update(json.dumps({"system": system, "tools": tools, "grader": grader,
-                         "tests": [{k: v for k, v in t.items() if k not in NOT_FINGERPRINTED} for t in tests]},
-                        sort_keys=True, default=str).encode())
+    what = {"system": system, "tools": tools, "grader": grader,
+            "tests": [{k: v for k, v in t.items() if k not in NOT_FINGERPRINTED} for t in tests]}
+    if max_tokens:
+        what["max_tokens"] = max_tokens
+    h.update(json.dumps(what, sort_keys=True, default=str).encode())
     for path in sorted(images):
         with open(path, "rb") as f:
             h.update(f.read())
@@ -217,13 +221,16 @@ def load_pack(path):
     if tools and "tools" not in needs:
         needs = list(needs) + ["tools"]
     gate = {"min_accuracy": 0.8, **meta.get("gate", {})}
+    max_tokens = meta.get("max_tokens", 0)
+    if "max_tokens" in meta and (isinstance(max_tokens, bool) or not isinstance(max_tokens, int) or max_tokens < 1):
+        raise PackError(f"{meta_path}: max_tokens must be a whole number above 0, not {max_tokens!r}")
     return Pack(name=name, path=path, label=meta.get("label", name), group=meta.get("group", "Other"),
                 description=meta.get("description", ""), grader=meta.get("grader", "answer"),
                 system=system, needs=list(needs), tools=tools, tests=tests,
-                fingerprint=_fingerprint(system, tools, meta.get("grader", "answer"), tests, images),
+                fingerprint=_fingerprint(system, tools, meta.get("grader", "answer"), tests, images, max_tokens),
                 gate=gate, screen=int(meta.get("screen", 10)),
                 certify_repeat=int(meta.get("certify", {}).get("repeat", 0)),
-                stop_early=bool(meta.get("stop_early", False)))
+                stop_early=bool(meta.get("stop_early", False)), max_tokens=max_tokens)
 
 
 def load_packs(packs_dir=None, errors=None):

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/src/tuieval/templates/packs/answer/pack.toml

@@ -6,6 +6,7 @@ grader = "answer"                # how answers are scored (answer, rag, reply, t
 system = "system.txt"            # the system prompt sent with every question
 order = 100                      # position in the list
 screen = 6                       # tests in a Screen run
+# max_tokens = 32768             # this pack's output budget, instead of models.toml [sampling] max_tokens
 
 [certify]
 repeat = 3                       # times each question is asked in a Certify run

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/src/tuieval/tui.py

@@ -531,7 +531,8 @@ class AnswerScreen(ModalScreen):
         partial = "  [cyan](pack still running or unfinished: not in scores yet)[/cyan]" if entry.get("partial") else ""
         quant = f"  [magenta]{rec['quantization']}[/magenta]" if rec.get("quantization") else ""
         finish = rec.get("finish") or "?"
-        finish = f"[bold red]{finish} (cut off at max_tokens)[/]" if finish == "length" else finish
+        limit = f"this pack's max_tokens, {pack.max_tokens:,}" if pack and pack.max_tokens else "max_tokens"
+        finish = f"[bold red]{finish} (cut off at {limit})[/]" if finish == "length" else finish
         toks = rec.get("completion_tokens")
         split = ", ".join(f"{k} {rec[k + '_tokens']:,}" for k in ("reasoning", "answer")
                           if rec.get(k + "_tokens") is not None)
@@ -709,6 +710,8 @@ class SetupScreen(Screen):
             count = (f"[b cyan]{len(picked)}/{len(pk.tests)} tests picked[/]" if picked
                      else f"[dim]{len(pk.tests)} tests[/dim]")
             needs = f"  [magenta]{', '.join(pk.needs)}[/magenta]" if pk.needs else ""
+            if pk.max_tokens:
+                needs += f"  [dim]max_tokens {pk.max_tokens:,}[/dim]"
             mix = {lvl: sum(t.get("difficulty") == lvl for t in pk.tests) for lvl in ("easy", "medium", "hard")}
             mixed = " ".join(f"[{DIFF_STYLE[l]}]{n}{l[0].upper()}[/]" for l, n in mix.items() if n)
             packs.add_option(Selection(f"[dim]{pk.group} ·[/dim] {pk.label}  {count} {mixed}{needs}",

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/tests/mock_server.py

@@ -9,6 +9,8 @@ fixed   always "ANSWER: 42"
 unavailable  HTTP 404 "No endpoints found" for every chat request
 
 Streams like llama.cpp: reasoning_content then content deltas, usage and timings at the end.
+An answer longer than the request's max_tokens (one token per word) is cut off with finish
+"length". --max-output N also serves an OpenRouter-style endpoints list with that output limit.
 """
 import argparse
 import json
@@ -47,7 +49,7 @@ def reply_for(test, mode):
 
 
 class Handler(BaseHTTPRequestHandler):
-    answers, mode, model, delay = {}, "oracle", "mock", 0.0
+    answers, mode, model, delay, max_output = {}, "oracle", "mock", 0.0, None
 
     def log_message(self, *a):
         pass
@@ -63,6 +65,9 @@ class Handler(BaseHTTPRequestHandler):
     def do_GET(self):
         if self.path.rstrip("/") in ("/v1/models", "/models"):
             return self._json({"data": [{"id": self.model}]})
+        if self.max_output and self.path.rstrip("/").endswith("/endpoints"):
+            return self._json({"data": {"endpoints": [{"tag": "mock/fp16", "quantization": "fp16", "status": 0,
+                                                       "max_completion_tokens": self.max_output}]}})
         if self.path == "/health":
             return self._json({"status": "ok", "model": self.model})
         self._json({"error": "not found"}, 404)
@@ -82,6 +87,9 @@ class Handler(BaseHTTPRequestHandler):
         if self.mode == "unavailable":   # like OpenRouter with no provider serving the model
             return self._json({"error": {"message": f"No endpoints found for {self.model}.", "code": 404}}, 404)
         text, calls = reply_for(test, self.mode)
+        finish = "tool_calls" if calls else "stop"
+        if req.get("max_tokens") and len(text.split()) > req["max_tokens"]:
+            text, finish = " ".join(text.split()[:req["max_tokens"]]), "length"
         self.send_response(200)
         self.send_header("Content-Type", "text/event-stream")
         self.end_headers()
@@ -98,7 +106,7 @@ class Handler(BaseHTTPRequestHandler):
             send({"choices": [{"delta": {"tool_calls": [{"index": i, "function": {
                 "name": c["name"], "arguments": json.dumps(c.get("arguments") or {})}}]}}]})
         words = max(1, len(text.split()) + 3)
-        send({"choices": [{"delta": {}, "finish_reason": "tool_calls" if calls else "stop"}],
+        send({"choices": [{"delta": {}, "finish_reason": finish}],
               "usage": {"prompt_tokens": len(str(content).split()), "completion_tokens": words},
               "timings": {"predicted_n": words, "predicted_per_second": 50.0, "prompt_n": 10,
                           "prompt_per_second": 500.0, "cache_n": 0}})
@@ -119,8 +127,10 @@ def main(argv=None):
     p.add_argument("--mode", choices=("oracle", "wrong", "fixed", "unavailable"), default="oracle")
     p.add_argument("--model", default="mock")
     p.add_argument("--delay", type=float, default=0.0, help="seconds each answer takes")
+    p.add_argument("--max-output", type=int)
     a, _ = p.parse_known_args(argv)   # other flags (e.g. speed knobs under test) are accepted and ignored
     Handler.answers, Handler.mode, Handler.model, Handler.delay = load_answers(a.packs), a.mode, a.model, a.delay
+    Handler.max_output = a.max_output
     srv = Server(("127.0.0.1", a.port), Handler)
     print(f"mock server on 127.0.0.1:{a.port} ({a.mode}, {len(Handler.answers)} known questions)", flush=True)
     try:

{tuieval-0.2.0.dev87 → tuieval-0.2.0.dev89}/tests/test_tuieval.py

@@ -49,10 +49,11 @@ def free_port():
 
 
 class Mock:
-    def __init__(self, packs, mode, port=None, delay=0.0):
+    def __init__(self, packs, mode, port=None, delay=0.0, *args):
         self.port = port or free_port()
         self.proc = subprocess.Popen([sys.executable, os.path.join(HERE, "mock_server.py"), "--port", str(self.port),
-                                      "--packs", packs, "--mode", mode, "--delay", str(delay)], stdout=subprocess.DEVNULL)
+                                      "--packs", packs, "--mode", mode, "--delay", str(delay), *args],
+                                     stdout=subprocess.DEVNULL)
         for _ in range(100):
             try:
                 urllib.request.urlopen(f"http://127.0.0.1:{self.port}/v1/models", timeout=1)
@@ -1815,6 +1816,109 @@ class FirstPack(unittest.TestCase):
         self.assertIn("→ fix the expected answer", out)
 
 
+class PackMaxTokens(unittest.TestCase):
+    """A pack's own max_tokens: sent with its requests, part of its fingerprint only, and packs that
+    can't fit it (context, a hosted endpoint's output limit) are skipped."""
+    LONG = " ".join(["word"] * 40) + "\nANSWER: 7"     # ~42 tokens at the mock's one token per word
+
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.ws = os.path.join(self.tmp.name, "ws")
+        tuieval(self.tmp.name, "init", self.ws)
+        for name, extra in (("short", "max_tokens = 20\n"), ("roomy", "max_tokens = 100\n"), ("plain", "")):
+            self.pack(name, extra)
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def pack(self, name, extra):
+        d = os.path.join(self.ws, "packs", name)
+        os.makedirs(d, exist_ok=True)
+        write(os.path.join(d, "pack.toml"), f'label = "{name}"\n{extra}[gate]\nmin_accuracy = 0.1\n')
+        write(os.path.join(d, "tests.yaml"), f"- id: q\n  input: {name} question\n  expected: 7\n"
+                                             f"  difficulty: easy\n  reference: {json.dumps(self.LONG)}\n")
+
+    def engine(self, code):
+        return subprocess.run([sys.executable, "-c", "from tuieval import engine, packs; e = engine.Engine(); " + code],
+                              capture_output=True, text=True, env=dict(os.environ, TUIEVAL_HOME=self.ws))
+
+    def result(self, label, pack):
+        path = os.path.join(self.ws, "results", label, f"{pack}.json")
+        return json.loads(read(path)) if os.path.isfile(path) else None
+
+    def test_load_and_fingerprint(self):
+        from tuieval import packs
+        d = os.path.join(self.ws, "packs", "plain")
+        before = packs.load_pack(d)
+        self.assertEqual(before.max_tokens, 0)
+        self.pack("plain", "")    # rewritten unchanged: same fingerprint as before the feature
+        self.assertEqual(packs.load_pack(d).fingerprint, before.fingerprint)
+        self.pack("plain", "max_tokens = 4096\n")
+        p = packs.load_pack(d)
+        self.assertEqual(p.max_tokens, 4096)
+        self.assertNotEqual(p.fingerprint, before.fingerprint)
+        self.pack("plain", "max_tokens = 8192\n")
+        self.assertNotEqual(packs.load_pack(d).fingerprint, p.fingerprint)
+        for bad in ("0", "-5", '"lots"', "1.5", "true"):
+            self.pack("plain", f"max_tokens = {bad}\n")
+            with self.assertRaises(packs.PackError, msg=bad):
+                packs.load_pack(d)
+        self.assertIn("max_tokens must be", tuieval(self.ws, "selftest", check=False).stdout)
+
+    def test_requests_use_the_packs_budget(self):
+        mock = Mock(os.path.join(self.ws, "packs"), "oracle")
+        try:
+            with open(os.path.join(self.ws, "models.toml"), "a") as f:
+                f.write(f'\n[servers.m]\nurl = "http://127.0.0.1:{mock.port}"\n')
+            tuieval(self.ws, "add", "mock", "--server", "m", "--label", "m")
+            tuieval(self.ws, "run", "--tier", "certify")
+            res = {p: self.result("m", p) for p in ("short", "roomy", "plain")}
+            self.assertEqual(res["short"]["results"][0]["finish"], "length")   # 42 tokens > 20
+            self.assertIn("TRUNCATED", res["short"]["results"][0]["reason"])
+            self.assertTrue(res["roomy"]["results"][0]["pass"])
+            self.assertTrue(res["plain"]["results"][0]["pass"])               # [sampling] max_tokens
+            self.assertEqual(res["short"]["pack"]["max_tokens"], 20)
+            self.assertNotIn("max_tokens", res["plain"]["pack"])
+            self.assertEqual(res["short"]["settings"], res["plain"]["settings"])   # model settings untouched
+
+            # changing one pack's budget makes only that pack's results outdated
+            self.pack("short", "max_tokens = 50\n")
+            out = self.engine("print(e.result_status('m', 'short')); print(e.result_status('m', 'plain'))").stdout
+            self.assertEqual(out.splitlines(), ["outdated: questions changed", "certified"])
+            tuieval(self.ws, "run", "--tier", "certify")
+            self.assertTrue(self.result("m", "short")["results"][0]["pass"])
+        finally:
+            mock.stop()
+
+    def test_skipped_when_the_context_is_too_small(self):
+        self.pack("short", "max_tokens = 16000\n")
+        mock = Mock(os.path.join(self.ws, "packs"), "oracle")
+        try:
+            with open(os.path.join(self.ws, "models.toml"), "a") as f:
+                f.write(f'\n[servers.m]\nurl = "http://127.0.0.1:{mock.port}"\n'
+                        '\n[[models]]\nlabel = "small"\nserver = "m"\nmodel = "mock"\nmax_context = 8192\n')
+            out = tuieval(self.ws, "run", "--tier", "certify").stdout
+        finally:
+            mock.stop()
+        self.assertIsNone(self.result("small", "short"))
+        self.assertIn("needs ~15k context", out)
+        self.assertIsNotNone(self.result("small", "plain"))
+
+    def test_skipped_when_the_endpoint_allows_fewer_output_tokens(self):
+        mock = Mock(os.path.join(self.ws, "packs"), "oracle", None, 0.0, "--max-output", "50")
+        try:
+            with open(os.path.join(self.ws, "models.toml"), "a") as f:
+                f.write(f'\n[servers.hosted]\nurl = "http://127.0.0.1:{mock.port}/v1"\npin_endpoint = true\n'
+                        '\n[[models]]\nlabel = "h"\nserver = "hosted"\nmodel = "mock"\n')
+            out = tuieval(self.ws, "run", "--tier", "certify").stdout
+        finally:
+            mock.stop()
+        self.assertIsNone(self.result("h", "roomy"))
+        self.assertIn("needs 100 output tokens, endpoint allows 50", out)
+        self.assertIsNotNone(self.result("h", "short"))
+        self.assertIsNotNone(self.result("h", "plain"))
+
+
 def fake_gguf(path, embedding=5120, size=0, kv_heads=4):
     """A GGUF header with just the keys machines.read_gguf reads, padded to size bytes. kv_heads: a
     number, or one per layer (a list, as some GGUFs store it)."""