tuieval 0.2.0.dev103__tar.gz → 0.2.0.dev105__tar.gz

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev105}/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix: an answer followed by a stray `</think>` (or a stray code fence after it) was graded as empty; the answer before the tag is graded now. The code grader also reads a doubled ```` ```python ```` fence as one.
 - Run screen: `x` removes the highlighted pack from the run going if it hasn't started (the reverse of adding it); a model left with nothing isn't started.
 - `k` (skip model) asks first, naming the model. Yes/no dialogs start on No; `y` answers yes, `n` or Esc no.
 - Fix: hosted models no longer count as running alongside local ones (they don't share the machine), so a one-model-at-a-time setup no longer gets the "ran alongside other models" note. Older results are read the same way.

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev105}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: tuieval
-Version: 0.2.0.dev103
+Version: 0.2.0.dev105
 Summary: Evaluate local and frontier LLMs on your own questions: accuracy, speed, tokens and PASS/FAIL verdicts, in the terminal.
 Project-URL: Homepage, https://github.com/ashe-wb/tuieval
 Project-URL: Issues, https://github.com/ashe-wb/tuieval/issues

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev105}/src/tuieval/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '0.2.0.dev103'
-__version_tuple__ = version_tuple = (0, 2, 0, 'dev103')
+__version__ = version = '0.2.0.dev105'
+__version_tuple__ = version_tuple = (0, 2, 0, 'dev105')
 
 __commit_id__ = commit_id = None

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev105}/src/tuieval/graders/__init__.py

@@ -123,8 +123,9 @@ def strip_thinking(text):
     """Safety net for servers that leave <think> blocks in the answer."""
     text = THINK_RE.sub("", text or "")
     if "</think>" in text:
-        text = text.split("</think>")[-1]
-    return text.strip()
+        before, after = text.rsplit("</think>", 1)
+        text = after if after.strip(" \t\r\n`") else before   # a stray closing tag (or fence) after the answer
+    return text.replace("<think>", "").replace("</think>", "").strip()
 
 
 def grade(test, answer, meta=None):

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev105}/src/tuieval/graders/code.py

@@ -15,8 +15,24 @@ from . import grader, result
 
 
 def _extract_code(text):
-    blocks = re.findall(r"```(?:python|py)?[ \t]*\n(.*?)```", text, re.DOTALL)
-    return blocks[-1] if blocks else text  # last block = the final answer, not a draft
+    """The last python (or untagged) fenced block: the final answer, not a draft. A doubled
+    opening fence is one fence, and a stray fence after the last block is ignored."""
+    blocks, lang, cur = [], None, None
+    for line in text.splitlines(keepends=True):
+        fence = line.strip()
+        if not fence.startswith("```"):
+            if cur is not None:
+                cur.append(line)
+        elif cur is None:
+            lang, cur = fence[3:].strip().lower(), []
+        elif fence[3:].strip() and not "".join(cur).strip():
+            lang = fence[3:].strip().lower()   # ```python twice
+        else:
+            if lang in ("", "python", "py", "python3"):
+                blocks.append("".join(cur))
+            cur = None
+    blocks = [b for b in blocks if b.strip()]
+    return blocks[-1] if blocks else text
 
 
 CHECK_RE = re.compile(r"^# (SETUP|CHECK: *(.+?)) *$")

{tuieval-0.2.0.dev103 → tuieval-0.2.0.dev105}/tests/test_tuieval.py

@@ -248,6 +248,21 @@ class WorkspaceGraders(unittest.TestCase):
 
 
 class Units(unittest.TestCase):
+    def test_strip_thinking_keeps_answer_before_stray_close_tag(self):
+        from tuieval.graders import strip_thinking
+        self.assertEqual(strip_thinking("<think>hm</think>\n42"), "42")
+        self.assertEqual(strip_thinking("leaked thoughts</think>\n42"), "42")
+        self.assertEqual(strip_thinking("```python\nx = 1\n```\n</think>\n"), "```python\nx = 1\n```")
+        self.assertEqual(strip_thinking("</think>"), "")
+        self.assertEqual(strip_thinking("```python\nx = 1\n```\n</think>\n```"), "```python\nx = 1\n```")
+
+    def test_extract_code_tolerates_doubled_and_stray_fences(self):
+        from tuieval.graders.code import _extract_code
+        self.assertEqual(_extract_code("```python\n```python\nx = 1\n```\n```"), "x = 1\n")
+        self.assertEqual(_extract_code("```py\nold\n```\ntext\n```python\nnew\n```"), "new\n")   # last block wins
+        self.assertEqual(_extract_code("```python\nx = 1\n```\n```bash\nrun it\n```"), "x = 1\n")
+        self.assertEqual(_extract_code("x = 1"), "x = 1")
+
     def test_settings_note_names_only_what_differs(self):
         from tuieval import compare
         infos = [{"label": m, "settings": {"temperature": 1.0, **extra}} for m, extra in