tuieval 0.2.0.dev87__py3-none-any.whl → 0.2.0.dev89__py3-none-any.whl

tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev87'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev87')
+__version__ = version = '0.2.0.dev89'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev89')
 
 __commit_id__ = commit_id = None

tuieval/engine.py

@@ -646,8 +646,9 @@ def serves(ids, name):
 
 
 def pack_min_context(pack):
-    """Rough context a pack needs: its longest prompt (~4 chars/token) plus room to answer."""
-    return max(len(t["input"]) for t in pack.tests) // 4 + 4096
+    """Rough context a pack needs: its longest prompt (~4 chars/token) plus room to answer (the
+    pack's own max_tokens when it sets one)."""
+    return max(len(t["input"]) for t in pack.tests) // 4 + (pack.max_tokens or 4096)
 
 
 class Cancelled(Exception):
@@ -872,7 +873,8 @@ class Engine:
                 needs_tools = m.get("tools") is not False and any("tools" in p.needs for p in self.packs.values())
                 ep = choose_endpoint(eps, m["model"], effective_sampling(self.cfg, m), needs_tools, previous)
                 ep = ep and {"tag": ep["tag"], "provider": ep.get("provider_name"),
-                             "quantization": ep.get("quantization") or "unknown"}
+                             "quantization": ep.get("quantization") or "unknown",
+                             "max_output": ep.get("max_completion_tokens")}
             except (OSError, ValueError):
                 # offline: assume the endpoint earlier results used (the run itself will need the network)
                 ep = {"tag": previous, "provider": None, "quantization": previous_quant} if previous else None
@@ -1196,6 +1198,9 @@ class Engine:
                 elif (sv.ctx or 10**9) < pack_min_context(pack):
                     job.status, job.note = "skipped", \
                         f"needs ~{pack_min_context(pack) // 1024}k context, {sv.ctx // 1024}k fits on {sv.machine.id}"
+                elif pack.max_tokens and pack.max_tokens > ((self.endpoint(m) or {}).get("max_output") or 10**9):
+                    job.status, job.note = "skipped", \
+                        f"needs {pack.max_tokens:,} output tokens, endpoint allows {self.endpoint(m)['max_output']:,}"
                 elif missing:
                     job.status, job.note = "skipped", f"{', '.join(missing)} not installed in tuieval's Python"
                 elif status and status.startswith("outdated"):
@@ -2037,7 +2042,8 @@ class Engine:
     def _body(self, job, test, rep=0):
         s = job.sampling
         body = {"model": job.model["served_name"], "messages": self._messages(job, test),
-                "temperature": s["temperature"], "top_p": s["top_p"], "max_tokens": s["max_tokens"]}
+                "temperature": s["temperature"], "top_p": s["top_p"],
+                "max_tokens": job.pack.max_tokens or s["max_tokens"]}
         for k in ("top_k", "min_p", "presence_penalty", "repeat_penalty", "seed", "reasoning_effort"):
             if k in s:
                 body[k] = s[k]
@@ -2380,7 +2386,8 @@ class Engine:
             "model": {**{k: m.get(k) for k in ("label", "served_name", "server", "model", "tags", "vision", "mmproj")},
                       **{k: m[k] for k in ("tools", "max_context", "thinking") if k in m}},
             "pack": {"name": job.pack.name, "label": job.pack.label, "group": job.pack.group,
-                     "fingerprint": job.pack.fingerprint, "tests": len(job.pack.tests)},
+                     "fingerprint": job.pack.fingerprint, "tests": len(job.pack.tests),
+                     **({"max_tokens": job.pack.max_tokens} if job.pack.max_tokens else {})},
             "settings": job.sampling,
             "serving": job.settings.get("serving"),
             "settings_fingerprint": settings_fingerprint(job.settings),

tuieval/packs.py

@@ -50,6 +50,7 @@ class Pack:
     screen: int            # tests in a screening run
     certify_repeat: int    # repeats in a certification run (0 = models.toml default)
     stop_early: bool = False   # stop the pack once its FAIL is certain (pack.toml stop_early)
+    max_tokens: int = 0    # output budget for this pack's answers (0 = [sampling] max_tokens)
 
     @property
     def modules(self):
@@ -135,12 +136,15 @@ NOT_FINGERPRINTED = {"reference", "wrong", "description", "category", "critical"
 DIFFICULTIES = ("easy", "medium", "hard")
 
 
-def _fingerprint(system, tools, grader, tests, images):
-    """Hash of what the model sees and what decides pass/fail."""
+def _fingerprint(system, tools, grader, tests, images, max_tokens=0):
+    """Hash of what the model sees and what decides pass/fail. A pack's own max_tokens counts
+    (it decides what gets cut off); packs without one keep the fingerprint they always had."""
     h = hashlib.sha256()
-    h.update(json.dumps({"system": system, "tools": tools, "grader": grader,
-                         "tests": [{k: v for k, v in t.items() if k not in NOT_FINGERPRINTED} for t in tests]},
-                        sort_keys=True, default=str).encode())
+    what = {"system": system, "tools": tools, "grader": grader,
+            "tests": [{k: v for k, v in t.items() if k not in NOT_FINGERPRINTED} for t in tests]}
+    if max_tokens:
+        what["max_tokens"] = max_tokens
+    h.update(json.dumps(what, sort_keys=True, default=str).encode())
     for path in sorted(images):
         with open(path, "rb") as f:
             h.update(f.read())
@@ -217,13 +221,16 @@ def load_pack(path):
     if tools and "tools" not in needs:
         needs = list(needs) + ["tools"]
     gate = {"min_accuracy": 0.8, **meta.get("gate", {})}
+    max_tokens = meta.get("max_tokens", 0)
+    if "max_tokens" in meta and (isinstance(max_tokens, bool) or not isinstance(max_tokens, int) or max_tokens < 1):
+        raise PackError(f"{meta_path}: max_tokens must be a whole number above 0, not {max_tokens!r}")
     return Pack(name=name, path=path, label=meta.get("label", name), group=meta.get("group", "Other"),
                 description=meta.get("description", ""), grader=meta.get("grader", "answer"),
                 system=system, needs=list(needs), tools=tools, tests=tests,
-                fingerprint=_fingerprint(system, tools, meta.get("grader", "answer"), tests, images),
+                fingerprint=_fingerprint(system, tools, meta.get("grader", "answer"), tests, images, max_tokens),
                 gate=gate, screen=int(meta.get("screen", 10)),
                 certify_repeat=int(meta.get("certify", {}).get("repeat", 0)),
-                stop_early=bool(meta.get("stop_early", False)))
+                stop_early=bool(meta.get("stop_early", False)), max_tokens=max_tokens)
 
 
 def load_packs(packs_dir=None, errors=None):

tuieval/templates/packs/answer/pack.toml

@@ -6,6 +6,7 @@ grader = "answer"                # how answers are scored (answer, rag, reply, t
 system = "system.txt"            # the system prompt sent with every question
 order = 100                      # position in the list
 screen = 6                       # tests in a Screen run
+# max_tokens = 32768             # this pack's output budget, instead of models.toml [sampling] max_tokens
 
 [certify]
 repeat = 3                       # times each question is asked in a Certify run

tuieval/tui.py

@@ -531,7 +531,8 @@ class AnswerScreen(ModalScreen):
         partial = "  [cyan](pack still running or unfinished: not in scores yet)[/cyan]" if entry.get("partial") else ""
         quant = f"  [magenta]{rec['quantization']}[/magenta]" if rec.get("quantization") else ""
         finish = rec.get("finish") or "?"
-        finish = f"[bold red]{finish} (cut off at max_tokens)[/]" if finish == "length" else finish
+        limit = f"this pack's max_tokens, {pack.max_tokens:,}" if pack and pack.max_tokens else "max_tokens"
+        finish = f"[bold red]{finish} (cut off at {limit})[/]" if finish == "length" else finish
         toks = rec.get("completion_tokens")
         split = ", ".join(f"{k} {rec[k + '_tokens']:,}" for k in ("reasoning", "answer")
                           if rec.get(k + "_tokens") is not None)
@@ -709,6 +710,8 @@ class SetupScreen(Screen):
             count = (f"[b cyan]{len(picked)}/{len(pk.tests)} tests picked[/]" if picked
                      else f"[dim]{len(pk.tests)} tests[/dim]")
             needs = f"  [magenta]{', '.join(pk.needs)}[/magenta]" if pk.needs else ""
+            if pk.max_tokens:
+                needs += f"  [dim]max_tokens {pk.max_tokens:,}[/dim]"
             mix = {lvl: sum(t.get("difficulty") == lvl for t in pk.tests) for lvl in ("easy", "medium", "hard")}
             mixed = " ".join(f"[{DIFF_STYLE[l]}]{n}{l[0].upper()}[/]" for l, n in mix.items() if n)
             packs.add_option(Selection(f"[dim]{pk.group} ·[/dim] {pk.label}  {count} {mixed}{needs}",

{tuieval-0.2.0.dev87.dist-info → tuieval-0.2.0.dev89.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev87
+Version: 0.2.0.dev89
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev87.dist-info → tuieval-0.2.0.dev89.dist-info}/RECORD

@@ -1,24 +1,24 @@
 tuieval/__init__.py,sha256=myVXVX7htbwezW6k3x_FuKgn1RSZ2WcTaDEMByGn6Wg,312
 tuieval/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
-tuieval/_version.py,sha256=MbDAskHZqL7Ly_g4fmQSqzeTJS3q2IGRhFiHFeJp5zA,535
+tuieval/_version.py,sha256=XiUzkOIwjrCyySM-wvAr3iW2t63wQjeKE_BTt6XJQng,535
 tuieval/builtin.toml,sha256=Kxw22RZCJhURUyu5Wz8tj5Y1nM85A8hFVRbA4ocoPUg,4293
 tuieval/cli.py,sha256=VwiaU1F9bZtU8C-gEowJq58kPeYQY4t6SY_HwrV-qgc,4702
 tuieval/client.py,sha256=kpsyLEZKfxCsvIGBHBbBy83TpI97CvnGCDq-4AbJS4Y,9986
 tuieval/compare.py,sha256=uXU_Yzf4zDSHoupHQrTmBhsfc8wCyI1nMaZQN81ybiA,30522
 tuieval/doctor.py,sha256=j2MdDORstsAGPnRmpsnxau1_fR6sTc_oEFGM5zugTJU,7246
-tuieval/engine.py,sha256=yMWvYq85WmYmk8fpMLRzYcxJxo7XcjEIUTa_y2VK7Kk,124538
+tuieval/engine.py,sha256=BS2BXLaxtw3RFI6-5PDvhV5VLh5v0m0cB_F3fPC5MtM,125105
 tuieval/export.py,sha256=dbHQgL_ubWtIPM1sLzAew_gETnrDJyd0zgqVA5vkpXM,9657
 tuieval/logs.py,sha256=01gZK6tf53dIThaIrs1bEsz9eqKYVlcxpyaLnSPTVxo,8127
 tuieval/machines.py,sha256=t9tu6CZeXZON4EtOVnEwP4VtaynpVLf9Cz8TNjf7jtY,20124
 tuieval/onboard.py,sha256=IYETVDWCCo9vF0Dtj4_rr9VanYeCky8PwP3-2g5A_l0,6215
-tuieval/packs.py,sha256=8bkF3xIDnWlt1JD2t3seHpMXW_RbMBgrCdIGEC-w-ZU,11458
+tuieval/packs.py,sha256=ZgaZ1m2ePYvDjmddSc3nafnMEn7kTq9WcDpCaJSZleQ,12026
 tuieval/profiles.py,sha256=soeLxjdB-l-OxSY8jZ4yjwfdSnsBV-QICcwKxRJS7nQ,3663
 tuieval/pta.py,sha256=i5oPj-Pz3N9u7bzzuZwsQvMGdxWdAncww2nR15K6PHY,12108
 tuieval/remove.py,sha256=Vwh43nm9lVb9U9aN-h6CMYbkTwdR3UWfZObwweF72n0,5157
 tuieval/run_evals.py,sha256=FU3bvu3jt9ZWg3YLsXNINbdeClXMioyc4zQ4yEhmBQs,47855
 tuieval/scaffold.py,sha256=YBOqniJpf_GtKKAFF3_OHxxMGbmTWdFjcClGcOnrrKE,12130
 tuieval/selftest.py,sha256=nExkrOidgymlmvkxImEBfrQG5iTUN5WuRMIKtqusCCs,9835
-tuieval/tui.py,sha256=ZE9FEOr84Vx2YULSiBpaZM4DyOo0iIkuKlH870K88I8,176810
+tuieval/tui.py,sha256=yD-3pQJ7I71r8lg49m0aWD5SlfA_Bk9BIz6XUJj8_UM,177016
 tuieval/tune.py,sha256=azxxthgJeag8JyoQUgaRxq6oI1XvEgzb9vkszjJZH7w,39645
 tuieval/verdict.py,sha256=3SM8qrVdWNIFDENagaR5ZPBYEnjsBtkYB9WS5TDVssU,20517
 tuieval/watch_proxy.py,sha256=bnDO1i6n6yUs1hODoUWcMppJMGPavoQqs6aNYrIxpT8,11416
@@ -31,7 +31,7 @@ tuieval/graders/rag.py,sha256=j_gEjmaXmY_hgKu7-7pVQpQ8Cwb49gadnwFurih-1Hw,1447
 tuieval/graders/reply.py,sha256=BdkL7EWSSMoyjwCOFa97TTQMR1QyrpMH-ntcgjzDPEg,2199
 tuieval/graders/tool_call.py,sha256=_QXURtOk4ZKBnxXVMzFL1c9jWLWksRlLZ4VqOCL0t90,3578
 tuieval/templates/models.toml,sha256=0RsHR_oCNmkkUdPBemLahNIwaOOPKqT97hA97TJFDYw,1021
-tuieval/templates/packs/answer/pack.toml,sha256=8bjMBxG1Zz3yxEnYCqRTNBJ-JGYlquqiK19p8URWwTc,1167
+tuieval/templates/packs/answer/pack.toml,sha256=8tKQZ-4y9p7I6LjtcJ6CKmqL8M8ZsPna0TT-6mWtmcI,1274
 tuieval/templates/packs/answer/system.txt,sha256=m4V6X1i49Bas2HowZMWgfdDSPDHTObb_WxHXt0xw1Yo,248
 tuieval/templates/packs/answer/tests.yaml,sha256=WiYPOknPcO9E6f1wXhLcfA0W5v-oK8MgZV956uWtcUQ,2660
 tuieval/templates/packs/code/pack.toml,sha256=nnrUVSWlIHAoORTIc4W3emoniwWOcfx2z-ipqIqU3ks,457
@@ -47,8 +47,8 @@ tuieval/templates/packs/tool_call/pack.toml,sha256=n4bx0lGJHSmW41uVqhfIXJ2BWZpmT
 tuieval/templates/packs/tool_call/system.txt,sha256=mKxu-lB-0YjLHXdgeFJa8x2C180OvcNz5cHv2N5Bg3U,254
 tuieval/templates/packs/tool_call/tests.yaml,sha256=Nv7c4syrzJ9cU8EBKSK_1ZSFx9V02aCjW4-2TBCPT7s,1859
 tuieval/templates/packs/tool_call/tools.yaml,sha256=HgxCkh0On2VwbHSsjAGM8tNCK7ZtKLGrs7lMCn3Fprs,1048
-tuieval-0.2.0.dev87.dist-info/METADATA,sha256=XHHaef6pqjG5DaNsjztrZJ9K_ZPhGi63ONwA_J8XFqg,14580
-tuieval-0.2.0.dev87.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-tuieval-0.2.0.dev87.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
-tuieval-0.2.0.dev87.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
-tuieval-0.2.0.dev87.dist-info/RECORD,,
+tuieval-0.2.0.dev89.dist-info/METADATA,sha256=t3hBuHdUYssk-1LBJ7YfWAHkTPjsTXR9CnErwPjDKo8,14580
+tuieval-0.2.0.dev89.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+tuieval-0.2.0.dev89.dist-info/entry_points.txt,sha256=7eIhi1GpFxBP2OLJV02Nn8LwsNuO512eKOGt1IBbj04,45
+tuieval-0.2.0.dev89.dist-info/licenses/LICENSE,sha256=7wfRgGtBGH1aXhZkIn47oUznqKC18kEPMvUVgol7Ffc,1077
+tuieval-0.2.0.dev89.dist-info/RECORD,,