tuieval 0.2.0.dev103__tar.gz → 0.2.0.dev104__tar.gz

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev104}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix: an answer followed by a stray `</think>` was graded as empty; the answer before the tag is graded now.
 - Run screen: `x` removes the highlighted pack from the run going if it hasn't started (the reverse of adding it); a model left with nothing isn't started.
 - `k` (skip model) asks first, naming the model. Yes/no dialogs start on No; `y` answers yes, `n` or Esc no.
 - Fix: hosted models no longer count as running alongside local ones (they don't share the machine), so a one-model-at-a-time setup no longer gets the "ran alongside other models" note. Older results are read the same way.

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev104}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev103
+Version: 0.2.0.dev104
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev104}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev103'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev103')
+__version__ = version = '0.2.0.dev104'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev104')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev104}/src/tuieval/graders/__init__.py

@@ -123,8 +123,9 @@ def strip_thinking(text):
     """Safety net for servers that leave <think> blocks in the answer."""
     text = THINK_RE.sub("", text or "")
     if "</think>" in text:
-        text = text.split("</think>")[-1]
-    return text.strip()
+        before, after = text.rsplit("</think>", 1)
+        text = after if after.strip() else before   # a stray closing tag after the answer
+    return text.replace("<think>", "").replace("</think>", "").strip()
 
 
 def grade(test, answer, meta=None):

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev104}/tests/test_tuieval.py

@@ -248,6 +248,13 @@ class WorkspaceGraders(unittest.TestCase):
 
 
 class Units(unittest.TestCase):
+    def test_strip_thinking_keeps_answer_before_stray_close_tag(self):
+        from tuieval.graders import strip_thinking
+        self.assertEqual(strip_thinking("<think>hm</think>\n42"), "42")
+        self.assertEqual(strip_thinking("leaked thoughts</think>\n42"), "42")
+        self.assertEqual(strip_thinking("```python\nx = 1\n```\n</think>\n"), "```python\nx = 1\n```")
+        self.assertEqual(strip_thinking("</think>"), "")
+
     def test_settings_note_names_only_what_differs(self):
         from tuieval import compare
         infos = [{"label": m, "settings": {"temperature": 1.0, **extra}} for m, extra in