tuieval 0.2.0.dev104__tar.gz → 0.2.0.dev107__tar.gz

{tuieval-0.2.0.dev104 → tuieval-0.2.0.dev107}/CHANGELOG.md

@@ -2,7 +2,8 @@
 
 ## Unreleased
 
-- Fix: an answer followed by a stray `</think>` was graded as empty; the answer before the tag is graded now.
+- Results → Failures: a Models button filters the list (the same pick as the other tabs), with the number of failures shown.
+- Fix: an answer followed by a stray `</think>` (or a stray code fence after it) was graded as empty; the answer before the tag is graded now. The code grader also reads a doubled ```` ```python ```` fence as one.
 - Run screen: `x` removes the highlighted pack from the run going if it hasn't started (the reverse of adding it); a model left with nothing isn't started.
 - `k` (skip model) asks first, naming the model. Yes/no dialogs start on No; `y` answers yes, `n` or Esc no.
 - Fix: hosted models no longer count as running alongside local ones (they don't share the machine), so a one-model-at-a-time setup no longer gets the "ran alongside other models" note. Older results are read the same way.

{tuieval-0.2.0.dev104 → tuieval-0.2.0.dev107}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev104
+Version: 0.2.0.dev107
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev104 → tuieval-0.2.0.dev107}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev104'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev104')
+__version__ = version = '0.2.0.dev107'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev107')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev104 → tuieval-0.2.0.dev107}/src/tuieval/graders/__init__.py

@@ -124,7 +124,7 @@ def strip_thinking(text):
     text = THINK_RE.sub("", text or "")
     if "</think>" in text:
         before, after = text.rsplit("</think>", 1)
-        text = after if after.strip() else before   # a stray closing tag after the answer
+        text = after if after.strip(" \t\r\n`") else before   # a stray closing tag (or fence) after the answer
     return text.replace("<think>", "").replace("</think>", "").strip()
 
 

{tuieval-0.2.0.dev104 → tuieval-0.2.0.dev107}/src/tuieval/graders/code.py

@@ -15,8 +15,24 @@ from . import grader, result
 
 
 def _extract_code(text):
-    blocks = re.findall(r"```(?:python|py)?[ \t]*\n(.*?)```", text, re.DOTALL)
-    return blocks[-1] if blocks else text  # last block = the final answer, not a draft
+    """The last python (or untagged) fenced block: the final answer, not a draft. A doubled
+    opening fence is one fence, and a stray fence after the last block is ignored."""
+    blocks, lang, cur = [], None, None
+    for line in text.splitlines(keepends=True):
+        fence = line.strip()
+        if not fence.startswith("```"):
+            if cur is not None:
+                cur.append(line)
+        elif cur is None:
+            lang, cur = fence[3:].strip().lower(), []
+        elif fence[3:].strip() and not "".join(cur).strip():
+            lang = fence[3:].strip().lower()   # ```python twice
+        else:
+            if lang in ("", "python", "py", "python3"):
+                blocks.append("".join(cur))
+            cur = None
+    blocks = [b for b in blocks if b.strip()]
+    return blocks[-1] if blocks else text
 
 
 CHECK_RE = re.compile(r"^# (SETUP|CHECK: *(.+?)) *$")

{tuieval-0.2.0.dev104 → tuieval-0.2.0.dev107}/src/tuieval/tui.py

@@ -454,7 +454,8 @@ with the model's reasoning and answer as they stream, then recent results with t
                          Results open here.
   Production readiness   one verdict per model and use case; the table says why and what to run next
   Per question           every question, model by model (filters: pack, level, Models disagree)
-  Failures               every wrong answer with the grader's reason; enter opens it in full
+  Failures               every wrong answer with the grader's reason (filter: Models); enter opens it
+                         in full
 
 Click a column name to sort a table by it; click it again to reverse. Empty cells stay last.
 
@@ -2825,6 +2826,9 @@ class ResultsScreen(Screen):
                 yield Static(id="pq-legend")
                 yield DataTable(id="per-question", zebra_stripes=True, cursor_type="cell")
             with TabPane("Failures", id="tab-failures"):
+                with Horizontal(id="fail-controls"):
+                    yield Button("Models: all", id="fail-models")
+                    yield Static(id="fail-count")
                 yield Static("[dim]Enter on a row shows the whole answer, its reasoning and what was checked.[/dim]")
                 yield DataTable(id="failures", zebra_stripes=True, cursor_type="row")
         yield Footer()
@@ -2902,12 +2906,7 @@ class ResultsScreen(Screen):
         info.update(f"{len(paths)} {'Smoke ' if smoke else ''}result files in {os.path.relpath(results_dir)}/" + banner
                     + "".join(f"\n[yellow]⚠ {n}[/yellow]" for n in notes))
         self.rows, self.infos = rows, infos
-        f = self.query_one("#failures", DataTable)
-        f.add_columns("Model", "Level", "Test", "Reason")
-        # compare.failures lists the failed rows in order, so they line up with fail_rows
-        self.fail_rows = [r for r in rows if not r["ok"]]
-        for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
-            f.add_row(model, diff_badge(level), rich_escape(test), rich_escape(reason[:130]), key=str(i))
+        self.fill_failures()
         self.fill_per_question()
         self.fill_pta()
         self.fill_readiness(results_dir)
@@ -2916,6 +2915,24 @@ class ResultsScreen(Screen):
             self.fill_insights()
         self.sort_tables()
 
+    def fill_failures(self):
+        """Every wrong answer, of the models picked (Models: shared with the other tabs)."""
+        f = self.query_one("#failures", DataTable)
+        f.clear(columns=True)
+        f.add_columns("Model", "Level", "Test", "Reason")
+        every = {r["model"] for r in self.rows}
+        chosen = {m for m in self.app.pq_models if m in every}
+        rows = [r for r in self.rows if not chosen or r["model"] in chosen]
+        # compare.failures lists the failed rows in order, so they line up with fail_rows
+        self.fail_rows = [r for r in rows if not r["ok"]]
+        for i, (model, test, reason, level) in enumerate(compare.failures(rows)):
+            f.add_row(model, diff_badge(level), rich_escape(test), rich_escape(reason[:130]), key=str(i))
+        self.query_one("#fail-models", Button).label = f"Models: {len(chosen)} picked" if chosen else "Models: all"
+        n = len(self.fail_rows)
+        self.query_one("#fail-count", Static).update(
+            f"{n} failure{'s' * (n != 1)}" + (f" of {len(chosen)} model{'s' * (len(chosen) != 1)}" if chosen else ""))
+        self.sort_table(f)
+
     @on(DataTable.RowSelected, "#failures")
     def failure_selected(self, event):
         # in the order shown, so [ and ] step through them as sorted
@@ -2979,6 +2996,7 @@ class ResultsScreen(Screen):
     @on(Button.Pressed, "#pq-models")
     @on(Button.Pressed, "#pta-models")
     @on(Button.Pressed, "#ins-models")
+    @on(Button.Pressed, "#fail-models")
     def pick_models(self):
         counts = {}
         for t in getattr(self, "pq_all_tests", []):
@@ -2997,6 +3015,7 @@ class ResultsScreen(Screen):
             # after the picker has closed: a table rebuilt under a modal keeps its header-only widths
             self.call_after_refresh(self.fill_per_question)
             self.call_after_refresh(self.fill_pta)
+            self.call_after_refresh(self.fill_failures)
             self.insights_stale = True
             if self.query_one(TabbedContent).active == "tab-insights":
                 self.call_after_refresh(self.fill_insights)
@@ -3397,6 +3416,9 @@ class EvalsApp(App):
     #pq-controls Checkbox { width: auto; }
     #pq-controls Select { width: 22; margin-right: 1; }
     #pq-models { margin-right: 1; min-width: 20; }
+    #fail-controls { height: 3; }
+    #fail-models { margin-right: 1; min-width: 20; }
+    #fail-count { width: auto; padding: 1 1; }
     #pq-compare-title { padding: 0 1; height: auto; }
     #pq-summary { height: auto; }   /* every picked model, no scrolling: the tab scrolls instead */
     #pq-legend { padding: 0 1; height: auto; }

{tuieval-0.2.0.dev104 → tuieval-0.2.0.dev107}/tests/test_tuieval.py

@@ -254,6 +254,14 @@ class Units(unittest.TestCase):
         self.assertEqual(strip_thinking("leaked thoughts</think>\n42"), "42")
         self.assertEqual(strip_thinking("```python\nx = 1\n```\n</think>\n"), "```python\nx = 1\n```")
         self.assertEqual(strip_thinking("</think>"), "")
+        self.assertEqual(strip_thinking("```python\nx = 1\n```\n</think>\n```"), "```python\nx = 1\n```")
+
+    def test_extract_code_tolerates_doubled_and_stray_fences(self):
+        from tuieval.graders.code import _extract_code
+        self.assertEqual(_extract_code("```python\n```python\nx = 1\n```\n```"), "x = 1\n")
+        self.assertEqual(_extract_code("```py\nold\n```\ntext\n```python\nnew\n```"), "new\n")   # last block wins
+        self.assertEqual(_extract_code("```python\nx = 1\n```\n```bash\nrun it\n```"), "x = 1\n")
+        self.assertEqual(_extract_code("x = 1"), "x = 1")
 
     def test_settings_note_names_only_what_differs(self):
         from tuieval import compare
@@ -1563,6 +1571,17 @@ class PTAIndex(unittest.TestCase):
                         app.screen.query_one(TabbedContent).active = "tab-failures"
                         await pilot.pause(0.3)
                         f = app.screen.query_one("#failures", DataTable)
+                        count = lambda: str(app.screen.query_one("#fail-count", Static).render())
+                        fails = {"all": [f.row_count, count()]}
+                        for m in ("good", "hosted"):                            # Models: only that model's failures
+                            app.pq_models = {m}
+                            app.screen.fill_failures()
+                            await pilot.pause(0.2)
+                            fails[m] = [f.row_count, count(), str(app.screen.query_one("#fail-models").label),
+                                        sorted({str(f.get_row_at(r)[0]) for r in range(f.row_count)})]
+                        app.pq_models = set()
+                        app.screen.fill_failures()
+                        await pilot.pause(0.2)
                         await click_header(f, "Test")
                         sorts["tests"], _ = await click_header(f, "Test")
                         sorts["tests"] = [str(f.get_row_at(r)[2]) for r in range(f.row_count)]
@@ -1573,7 +1592,7 @@ class PTAIndex(unittest.TestCase):
                         s = app.screen
                         sorts["opened"] = s.entries[s.index]["record"]["test"]
                         print(json.dumps({"rows": rows, "bars": "P parsimony" in tri, "disagree": [both, alone],
-                                          "tab": tab, "sorts": sorts}))
+                                          "tab": tab, "sorts": sorts, "fails": fails}))
                 asyncio.run(go())
             """)
             p = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, timeout=300,
@@ -1588,6 +1607,10 @@ class PTAIndex(unittest.TestCase):
             self.assertEqual((s["up"], s["down"], s["kept"]), (["hosted", "good"], ["good", "hosted"], ["good", "hosted"]))
             self.assertIn("answers right ▼", s["labels"])
             self.assertEqual((s["tests"][0], s["opened"]), ("apps: d", "d"))
+            fails = got["fails"]
+            self.assertEqual(fails["all"], [4, "4 failures"])
+            self.assertEqual(fails["good"], [0, "0 failures of 1 model", "Models: 1 picked", []])
+            self.assertEqual(fails["hosted"], [4, "4 failures of 1 model", "Models: 1 picked", ["hosted"]])
 
 
 class Unavailable(unittest.TestCase):