tuieval 0.2.0.dev126__tar.gz → 0.2.0.dev129__tar.gz

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev129}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Results: picking a pack (PTA index or Per question) compares the models that answered all of it; the Models button can change that, and All packs goes back to the earlier pick.
 - Fix: a request that runs out of time says `time limit: timed out after N s`, not `connection error`, and `tuieval compare` counts it against the model like the run does (it was left out as an error).
 - Agent packs (`kind = "agent"`): each test is a small project with a task, worked on by a command-line agent you define in `[agents]`, using the model tuieval serves, and graded by the task's visible and hidden tests. Results show the agent's stats, diff and transcript; `tuieval compare` adds an Agent tasks table; `tuieval selftest` checks each task's reference and wrong fixes. See docs/agent-packs.md.
 - Answers that used more tokens than the `max_tokens` sent (a provider that didn't stop them) are marked over budget: `(n over budget)` in the PTA index, an **over budget** column, `over` in Per question and a note in the answer view. `[defaults] over_budget = "fail"` fails them instead. Works on stored results.

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev129}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev126
+Version: 0.2.0.dev129
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev129}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev126'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev126')
+__version__ = version = '0.2.0.dev129'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev129')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev129}/src/tuieval/compare.py

@@ -390,6 +390,17 @@ def with_models(test, models):
     return dict(test, cells=cells, spread=_spread(cells))
 
 
+def complete_models(rows, suite, test_ids=None):
+    """The models whose rows answer every question of a pack: test_ids (the pack's current tests),
+    or every question of it seen in the rows. Partial rows (a pack still running) don't count."""
+    rows = [r for r in rows if r["suite"] == suite and not r.get("partial")]
+    wanted = {f"{suite}: {t}" for t in test_ids} if test_ids is not None else {r["test"] for r in rows}
+    answered = {}
+    for r in rows:
+        answered.setdefault(r["model"], set()).add(r["test"])
+    return sorted(m for m, tests in answered.items() if wanted and wanted <= tests)
+
+
 def compare_models(tests, models):
     """Per model over per_question entries that all `models` answered: passes, token and time
     totals/medians (totals add each question's median over repeats, so repeats don't inflate them), questions where it used the fewest tokens / was fastest (among passing

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev129}/src/tuieval/tui.py

@@ -2963,6 +2963,9 @@ class ResultsScreen(Screen):
         self.rows, self.infos, self.partial_rows, self.superseded = [], [], [], set()
         if not hasattr(self.app, "pq_models"):
             self.app.pq_models = set()
+        if getattr(self.app, "pack_pick", None):   # the pack lists open on All packs again
+            self.app.pq_models = self.app.pack_pick["before"]
+        self.app.pack_pick = None
         if not hasattr(self.app, "result_sorts"):
             self.app.result_sorts = {}   # table id -> (column name, descending), kept for this session
         self.fill()
@@ -3054,7 +3057,7 @@ class ResultsScreen(Screen):
         for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
             when = dates.day(dates.answered_at(self.fail_rows[i].get("raw") or {}))
             f.add_row(model, diff_badge(level), rich_escape(test), when, rich_escape(reason[:130]), key=str(i))
-        self.query_one("#fail-models", Button).label = f"Models: {len(chosen)} picked" if chosen else "Models: all"
+        self.query_one("#fail-models", Button).label = self.models_label(chosen)
         n = len(self.fail_rows)
         self.query_one("#fail-count", Static).update(
             f"{n} failure{'s' * (n != 1)}" + (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
@@ -3072,7 +3075,6 @@ class ResultsScreen(Screen):
         self.app.push_screen(AnswerScreen(keep, keep.index(entries[i]), what="failure"))
 
     # -- per question: tokens and time for every model on the same question
-    @on(Select.Changed, "#pq-pack")
     @on(Select.Changed, "#pq-level")
     @on(Checkbox.Changed, "#pq-partial")
     @on(Checkbox.Changed, "#pq-shared")
@@ -3214,7 +3216,7 @@ class ResultsScreen(Screen):
         finally:
             self.filling_insights = False
         chosen = {m for m in self.app.pq_models if m in every}
-        self.query_one("#ins-models", Button).label = f"Models: {len(chosen)} picked" if chosen else "Models: all"
+        self.query_one("#ins-models", Button).label = self.models_label(chosen)
         main = os.path.abspath(self.results_dir or e.results_dir) == os.path.abspath(e.results_dir)
         bases = {m["label"]: m["base"] for m in e.cfg["models"] if m.get("base")}
         by = self.query_one("#ins-cost", Select).value or "tokens"
@@ -3222,9 +3224,46 @@ class ResultsScreen(Screen):
         body.update("\n\n".join(insights.render(p, i) for i, p in enumerate(out, 1)))
 
     @on(Select.Changed, "#pta-pack")
-    def pta_changed(self):
-        if getattr(self, "rows", None) is not None:
-            self.fill_pta()
+    @on(Select.Changed, "#pq-pack")
+    def pack_changed(self, event):
+        """A pack picked (PTA index or Per question, kept the same in both): compare the models that
+        answered every question of it (the Models button can change that); All packs: back to the
+        models picked before."""
+        if getattr(self, "rows", None) is None:
+            return
+        e, pack = self.app.engine, event.value or ""
+        for other, values in (("#pta-pack", "pta_pack_values"), ("#pq-pack", "pq_pack_values")):
+            sel = self.query_one(other, Select)
+            if sel is not event.select and sel.value != pack and pack in getattr(self, values, [""]):
+                with sel.prevent(Select.Changed):
+                    sel.value = pack
+        shared = self.query_one("#pq-shared", Checkbox)
+        p = self.app.pack_pick or {"before": set(self.app.pq_models), "shared": shared.value}
+        before = p["before"]
+        if not pack:
+            self.app.pack_pick = None
+            with shared.prevent(Checkbox.Changed):   # set_models may have ticked it
+                shared.value = p["shared"]
+            self.set_models(before)
+            return
+        rows = [r for r in self.rows if r.get("path") not in self.superseded]
+        complete = set(compare.complete_models(rows, pack, [t["id"] for t in e.packs[pack].tests]
+                                               if pack in e.packs else None))
+        self.app.pack_pick = {"pack": pack, "models": complete, "before": before, "shared": p["shared"]}
+        if not complete:
+            self.notify(f"No model has answered every question of {self.pack_label(pack)} yet: "
+                        "the models picked before are kept.", severity="warning", timeout=8)
+        self.set_models(complete or before)
+
+    def pack_label(self, pack):
+        return self.app.engine.packs[pack].label if pack in self.app.engine.packs else pack
+
+    def models_label(self, chosen):
+        """The Models buttons' text: the models a pack pick chose say so."""
+        p = self.app.pack_pick
+        if chosen and p and chosen == p["models"]:
+            return f"Models: {len(chosen)} complete on {self.pack_label(p['pack'])}"
+        return f"Models: {len(chosen)} picked" if chosen else "Models: all"
 
     def fill_pta(self):
         """The PTA index (pta.py) of the picked models (Models button, shared with Per question) on
@@ -3245,7 +3284,7 @@ class ResultsScreen(Screen):
             rows = [r for r in rows if r["suite"] == sel.value]
         present = {r["model"] for r in rows}
         chosen = {m for m in self.app.pq_models if m in present}
-        self.query_one("#pta-models", Button).label = f"Models: {len(chosen)} picked" if chosen else "Models: all"
+        self.query_one("#pta-models", Button).label = self.models_label(chosen)
         t = self.query_one("#pta", DataTable)
         t.clear(columns=True)
         if not present:
@@ -3300,8 +3339,7 @@ class ResultsScreen(Screen):
         all_tests = compare.per_question(rows)
         self.pq_all_tests = all_tests
         chosen = {m for m in self.app.pq_models if any(m in t["cells"] for t in all_tests)}
-        self.query_one("#pq-models", Button).label = (
-            f"Models: {len(chosen)} picked" if chosen else "Models: all")
+        self.query_one("#pq-models", Button).label = self.models_label(chosen)
         tests = []
         for t in all_tests:
             if (pack and t["suite"] != pack) or (level and t["difficulty"] != level):

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev129}/tests/mock_server.py

@@ -56,6 +56,7 @@ class Handler(BaseHTTPRequestHandler):
     answers, mode, model, delay, max_output = {}, "oracle", "mock", 0.0, None
     stall, stall_before, silent = 0.0, 0.0, False
     die_after = None
+    hold, answered = None, 0   # while the hold file exists, answers after the first wait (a run that stays going)
 
     def log_message(self, *a):
         pass
@@ -120,6 +121,8 @@ class Handler(BaseHTTPRequestHandler):
                     self.wfile.flush()
                 time.sleep(0.2)
 
+        while self.hold and Handler.answered and os.path.exists(self.hold):
+            wait(0.2)
         wait(self.stall_before)
         send({"choices": [{"delta": {"reasoning_content": "Thinking it over. "}}]})
         wait(self.stall)
@@ -136,6 +139,7 @@ class Handler(BaseHTTPRequestHandler):
                           "prompt_per_second": 500.0, "cache_n": 0}})
         die_if_due()
         self.wfile.write(b"data: [DONE]\n\n")
+        Handler.answered += 1
 
 
 class Server(ThreadingHTTPServer):
@@ -157,11 +161,12 @@ def main(argv=None):
     p.add_argument("--stall-before", type=float, default=0.0)
     p.add_argument("--silent", action="store_true")
     p.add_argument("--die-after", type=int)
+    p.add_argument("--hold", help="while this file exists, every answer after the first waits")
     a, _ = p.parse_known_args(argv)   # other flags (e.g. speed knobs under test) are accepted and ignored
     Handler.answers, Handler.mode, Handler.model, Handler.delay = load_answers(a.packs), a.mode, a.model, a.delay
     Handler.max_output = a.max_output
     Handler.stall, Handler.stall_before, Handler.silent = a.stall, a.stall_before, a.silent
-    Handler.die_after = a.die_after
+    Handler.die_after, Handler.hold = a.die_after, a.hold
     srv = Server(("127.0.0.1", a.port), Handler)
     print(f"mock server on 127.0.0.1:{a.port} ({a.mode}, {len(Handler.answers)} known questions)", flush=True)
     try:

{tuieval-0.2.0.dev126 → tuieval-0.2.0.dev129}/tests/test_tuieval.py

@@ -1412,8 +1412,12 @@ class AddToRun(unittest.TestCase):
 
     def test_tui_never_queues_the_same_thing_twice(self):
         """Start while a run is going never adds or queues a model and pack already running or queued."""
+        hold = os.path.join(self.tmp.name, "hold")   # the first run stays going until every pick is made
+        write(hold, "")
+        path = os.path.join(self.ws, "models.toml")
+        write(path, read(path).replace('"--delay", "0.2"', f'"--delay", "0.2", "--hold", {json.dumps(hold)}'))
         code = textwrap.dedent("""
-            import asyncio, json, time
+            import asyncio, json, os, time
             from tuieval.tui import EvalsApp, RunScreen, SetupScreen, ChoiceScreen
             from textual.widgets import SelectionList
             async def pick(app, pilot, models, packs=("apps",)):
@@ -1459,6 +1463,7 @@ class AddToRun(unittest.TestCase):
                     if isinstance(s, ChoiceScreen):
                         s.dismiss("queue")
                     await pilot.pause()
+                    os.remove(os.environ["HOLD"])
                     deadline = time.time() + 300   # the queued runs start one after another: wait for all three
                     while time.time() < deadline:
                         await pilot.pause(0.2)
@@ -1469,7 +1474,7 @@ class AddToRun(unittest.TestCase):
             asyncio.run(go())
         """)
         p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=600,
-                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS=""))
+                           env=dict(os.environ, TUIEVAL_HOME=self.ws, TUIEVAL_DETECT_PORTS="", HOLD=hold))
         out = json.loads(p.stdout.strip().splitlines()[-1]) if p.stdout.strip() else p.stderr
         self.assertIsInstance(out, dict, out)
         self.assertEqual(out["same"], [False, 0])
@@ -1651,6 +1656,83 @@ class PTAIndex(unittest.TestCase):
         self.assertEqual(sorted(cells, key=sort_key), ["10/20", "3/4", "4/4", "abc", "-"])
         self.assertEqual(sorted(cells, key=lambda c: sort_key(c, True)), ["4/4", "3/4", "10/20", "abc", "-"])
 
+    def test_complete_models(self):
+        from tuieval import compare
+        rows = [{"model": m, "suite": s, "test": f"{s}: {t}", **({"partial": True} if m == "running" else {})}
+                for m, s, ts in (("full", "p", "abc"), ("half", "p", "ab"), ("running", "p", "abc"), ("full", "o", "a"))
+                for t in ts]
+        self.assertEqual(compare.complete_models(rows, "p", ["a", "b", "c"]), ["full"])   # half misses c; running is ◐
+        self.assertEqual(compare.complete_models(rows, "p", ["a", "b"]), ["full", "half"])
+        self.assertEqual(compare.complete_models(rows, "p", ["a", "b", "c", "d"]), [])     # a question added since
+        self.assertEqual(compare.complete_models(rows, "p"), ["full"])                     # pack gone: questions seen
+        self.assertEqual(compare.complete_models(rows, "x", []), [])
+
+    def test_results_pack_picks_the_models_that_answered_all_of_it(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            ws = os.path.join(tmp, "ws")
+            tuieval(tmp, "init", ws)
+            for name, ids in (("apps", "abcd"), ("other", "ab")):
+                d = os.path.join(ws, "packs", name)
+                os.makedirs(d)
+                write(os.path.join(d, "pack.toml"), f'label = "{name.title()}"\n[certify]\nrepeat = 1\n')
+                write(os.path.join(d, "tests.yaml"), "".join(
+                    f"- {{id: {t}, input: question {t}, expected: 1, reference: 'ANSWER: 1', wrong: ['ANSWER: 2'], "
+                    "difficulty: easy}\n" for t in ids))
+            mock = Mock(os.path.join(ws, "packs"), "oracle")
+            try:
+                with open(os.path.join(ws, "models.toml"), "a") as f:
+                    f.write(f'\n[servers.here]\nurl = "http://127.0.0.1:{mock.port}"\n')
+                for label in ("full", "half", "solo"):
+                    tuieval(ws, "add", "mock", "--server", "here", "--label", label)
+                tuieval(ws, "run", "--tier", "certify", "--only", "full")
+                tuieval(ws, "run", "--tier", "certify", "--only", "half", "--packs", "apps", "--tests", "a,b")
+                tuieval(ws, "run", "--tier", "certify", "--only", "solo", "--packs", "other")
+            finally:
+                mock.stop()
+            code = textwrap.dedent("""
+                import asyncio
+                from tuieval.tui import EvalsApp, ResultsScreen
+                from textual.widgets import Button, DataTable, Select
+                async def go():
+                    app = EvalsApp({})
+                    async with app.run_test(size=(180, 50)) as pilot:
+                        await pilot.pause()
+                        app.pq_models = {"half"}                      # picked by hand before
+                        app.push_screen(ResultsScreen())
+                        await pilot.pause(0.5)
+                        s = app.screen
+                        def state():
+                            pq = s.query_one("#per-question", DataTable)
+                            cols = [str(c.label) for c in pq.ordered_columns][len(s.PQ_FIXED):]
+                            return (sorted(app.pq_models), s.query_one("#pta-pack", Select).value,
+                                    str(s.query_one("#pq-models", Button).label), cols, s.query_one("#pq-shared").value)
+                        s.query_one("#pq-pack", Select).value = "apps"
+                        await pilot.pause(0.3)
+                        print(state())
+                        s.query_one("#pta-pack", Select).value = "other"
+                        await pilot.pause(0.3)
+                        print(state())
+                        s.query_one("#pta-pack", Select).value = ""
+                        await pilot.pause(0.3)
+                        print(state())
+                        app.pq_models = set()                         # none picked: All packs gives every model again
+                        s.set_models(set())
+                        await pilot.pause(0.3)
+                        for pack in ("apps", ""):
+                            s.query_one("#pq-pack", Select).value = pack
+                            await pilot.pause(0.3)
+                            print(state())
+                asyncio.run(go())
+            """)
+            out = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True,
+                                 env=dict(os.environ, TUIEVAL_HOME=ws, TUIEVAL_DETECT_PORTS=""), timeout=60)
+            self.assertEqual(out.stdout.strip().splitlines(), [
+                "(['full'], 'apps', 'Models: 1 complete on Apps', ['full'], False)",
+                "(['full', 'solo'], 'other', 'Models: 2 complete on Other', ['full', 'solo'], False)",
+                "(['half'], '', 'Models: 1 picked', ['half'], False)",
+                "(['full'], 'apps', 'Models: 1 complete on Apps', ['full'], True)",     # none picked before: ticked
+                "([], '', 'Models: all', ['full', 'half', 'solo'], False)"], out.stderr[-2000:])
+
     def test_cli_report_and_tui(self):
         with tempfile.TemporaryDirectory() as tmp:
             ws = os.path.join(tmp, "ws")