verifiers 0.1.15.dev414__tar.gz → 0.1.15.dev416__tar.gz

{verifiers-0.1.15.dev414 → verifiers-0.1.15.dev416}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev414
+Version: 0.1.15.dev416
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.1.15.dev416/tests/v1/test_scoring.py

@@ -0,0 +1,33 @@
+import verifiers.v1 as vf
+
+
+def test_compare_stdout_results_accepts_token_equal_text() -> None:
+    assert vf.compare_stdout_results("hello   world\n", "hello world\n")
+
+
+def test_compare_stdout_results_keeps_numeric_tolerance() -> None:
+    assert vf.compare_stdout_results("1.0001 2.0\n", "1.0002 2.0009\n")
+
+
+def test_parse_pytest_outcomes_strips_xfail_xpass_reasons() -> None:
+    output = "\n".join(
+        [
+            "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked",
+            "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed",
+            "FAILED tests/test_mod.py::test_param[a - b] - assert left - right",
+            "PASSED tests/test_mod.py::test_ok",
+        ]
+    )
+
+    assert vf.parse_pytest_outcomes(output) == {
+        "tests/test_mod.py::test_xfail": "XFAIL",
+        "tests/test_mod.py::test_xpass": "XPASS",
+        "tests/test_mod.py::test_param[a - b]": "FAILED",
+        "tests/test_mod.py::test_ok": "PASSED",
+    }
+
+
+def test_parse_judge_choice_uses_first_choice_after_verdict_marker() -> None:
+    response = "Final Judgment: B because it is a better answer"
+
+    assert vf.parse_judge_choice(response, choices=("A", "B")) == "B"

{verifiers-0.1.15.dev414 → verifiers-0.1.15.dev416}/verifiers/v1/__init__.py

@@ -49,6 +49,14 @@ from verifiers.v1.loaders import (
     task_type,
     taskset_config_type,
 )
+from verifiers.v1.scoring import (
+    compare_stdout_results as compare_stdout_results,
+    extract_boxed_answer as extract_boxed_answer,
+    parse_judge_choice as parse_judge_choice,
+    parse_pytest_outcomes as parse_pytest_outcomes,
+    read_answer_file_or_last_reply as read_answer_file_or_last_reply,
+    verify_boxed_math_answer as verify_boxed_math_answer,
+)
 from verifiers.v1.retries import RetryConfig, RolloutRetryConfig
 from verifiers.v1.rollout import Rollout
 from verifiers.v1.runtimes import (
@@ -195,6 +203,13 @@ __all__ = [
     "JudgeConfig",
     "JudgeSamplingConfig",
     "JudgeResponse",
+    # scoring
+    "compare_stdout_results",
+    "extract_boxed_answer",
+    "parse_judge_choice",
+    "parse_pytest_outcomes",
+    "read_answer_file_or_last_reply",
+    "verify_boxed_math_answer",
     # mcp
     "Toolset",
     "ToolsetConfig",

verifiers-0.1.15.dev416/verifiers/v1/cli/dashboard/base.py

@@ -0,0 +1,108 @@
+"""The shared dashboard engine: a rich `Live` view on a fixed refresh tick.
+
+The eval and validate dashboards each build their own frame; `live_view` just drives whichever
+`render` it's given — refreshing on a timer and drawing a final frame on exit. When given an
+`on_key` handler it also reads left/right arrow presses from the terminal (see `_key_reader`).
+"""
+
+import asyncio
+import contextlib
+import os
+import sys
+from collections.abc import Callable, Iterator
+
+from rich.console import Group
+from rich.live import Live
+
+try:  # POSIX-only terminal control; absent on Windows, where key reading is skipped.
+    import termios
+    import tty
+
+    _HAS_TTY = True
+except ImportError:
+    _HAS_TTY = False
+
+# Arrow-key escape sequences → direction, covering both the normal (`ESC [`) and application
+# (`ESC O`) cursor-key modes a terminal might send.
+_ARROWS = {
+    b"\x1b[C": "right",
+    b"\x1bOC": "right",
+    b"\x1b[D": "left",
+    b"\x1bOD": "left",
+}
+
+
+@contextlib.contextmanager
+def _key_reader(
+    on_key: Callable[[str], None] | None, refresh: Callable[[], None]
+) -> Iterator[None]:
+    """Dispatch left/right arrow presses to `on_key`, redrawing immediately after each so the
+    view feels instant rather than waiting for the next refresh tick. A no-op unless `on_key` is
+    given and stdin is a real terminal. Puts stdin in cbreak mode (char-at-a-time, no echo, Ctrl+C
+    still interrupts) and reads it via the event loop — no extra thread — restoring the terminal on
+    exit."""
+    if on_key is None or not _HAS_TTY or not sys.stdin.isatty():
+        yield
+        return
+    fd = sys.stdin.fileno()
+    loop = asyncio.get_running_loop()
+    old = termios.tcgetattr(fd)
+    # carries a partial escape across reads — cbreak can split a key's 3 bytes over callbacks
+    buf = bytearray()
+
+    def _read() -> None:
+        try:
+            data = os.read(fd, 32)
+        except OSError:
+            return
+        if not data:
+            return
+        buf.extend(data)
+        fired = False
+        i = 0
+        while i < len(buf):  # a chunk may hold several presses; scan the whole buffer
+            if key := _ARROWS.get(bytes(buf[i : i + 3])):
+                on_key(key)
+                fired = True
+                i += 3
+            elif buf[i] == 0x1B and len(buf) - i < 3:
+                break  # incomplete escape at the tail — hold it for the next read
+            else:
+                i += 1
+        del buf[:i]
+        if fired:
+            refresh()
+
+    # setcbreak inside the try so the finally always restores the terminal, even if it (or
+    # add_reader) raises — otherwise stdin could be left in cbreak with echo off after we exit.
+    try:
+        tty.setcbreak(fd)
+        loop.add_reader(fd, _read)
+        yield
+    finally:
+        loop.remove_reader(fd)
+        termios.tcsetattr(fd, termios.TCSADRAIN, old)
+
+
+@contextlib.asynccontextmanager
+async def live_view(
+    render: Callable[[], Group], on_key: Callable[[str], None] | None = None
+):
+    """Refresh `render()` every 0.25s until the block exits, then draw one final frame. When
+    `on_key` is given, left/right arrow presses are dispatched to it (and redraw at once)."""
+    with Live(render(), auto_refresh=False) as live:
+
+        async def loop() -> None:
+            while True:
+                await asyncio.sleep(0.25)
+                live.update(render(), refresh=True)
+
+        task = asyncio.create_task(loop())
+        with _key_reader(on_key, lambda: live.update(render(), refresh=True)):
+            try:
+                yield
+            finally:
+                task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await task
+                live.update(render(), refresh=True)

{verifiers-0.1.15.dev414 → verifiers-0.1.15.dev416}/verifiers/v1/cli/dashboard/eval.py

@@ -220,8 +220,8 @@ def Progress(
         f"{len(done)}/{len(rollouts)} · {format_time(time.time() - start)} · "
         f"reward {reward} · err {err}"
     )
-    if page is not None:  # rollouts overflow the screen — show which page is on screen
-        stats += f"  (page {page[0]}/{page[1]})"
+    if page is not None:  # overflowing — show which page, and that the arrows page
+        stats += f"  (page {page[0]}/{page[1]} ◄ ►)"
     row = Table.grid(expand=True, padding=(0, 1))
     row.add_column(ratio=1)  # bar stretches to fill the width left of the stats
     row.add_column(justify="right", no_wrap=True)
@@ -495,13 +495,42 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
     return grid
 
 
+class Pager:
+    """Which page of overflowing rollout rows is on screen. Auto-advances on a timer until the
+    user takes over with the left/right arrows, after which it stays where they leave it. `count`
+    (the page count, set each render by `_paginate`) gates the arrows: they're inert while a single
+    page fits, so a stray press before rollouts overflow can't switch off auto-advance or offset the
+    starting page once paging begins. The chosen page is clamped to `count` (it can shrink on a
+    resize; it otherwise only grows, as rollouts are never removed)."""
+
+    def __init__(self) -> None:
+        self.page = 0
+        self.manual = False
+        self.count = 1
+
+    def on_key(self, key: str) -> None:
+        if key in ("left", "right") and self.count > 1:
+            self.manual = True
+            self.page += 1 if key == "right" else -1
+
+    def index(self, now: float) -> int:
+        # Track the auto page while it drives, so the first arrow continues from what's on screen
+        # rather than jumping back to page 1. Clamp in manual mode (count can shrink on resize).
+        if not self.manual:
+            self.page = int(now / _PAGE_SECONDS) % self.count
+        else:
+            self.page = max(0, min(self.page, self.count - 1))
+        return self.page
+
+
 def _paginate(
-    groups: list[list[Rollout]], rows_per_page: int, now: float
+    groups: list[list[Rollout]], rows_per_page: int, pager: Pager, now: float
 ) -> tuple[list[list[Rollout]], int, int]:
     """Pack groups (a task's rollouts kept together) into pages of at most `rows_per_page` rows,
-    cycling to the next page every `_PAGE_SECONDS`. Returns (this page's groups, 0-based index,
-    page count) — a single page when everything already fits."""
+    selecting the one `pager` points at. Returns (this page's groups, 0-based index, page count) —
+    a single page when everything already fits."""
     if sum(len(g) for g in groups) <= rows_per_page:
+        pager.count = 1  # everything fits on one page — arrows stay inert
         return groups, 0, 1
     pages: list[list[list[Rollout]]] = []
     current: list[list[Rollout]] = []
@@ -514,20 +543,24 @@ def _paginate(
         used += len(group)
     if current:
         pages.append(current)
-    index = int(now / _PAGE_SECONDS) % len(pages)
+    pager.count = len(pages)
+    index = pager.index(now)
     return pages[index], index, len(pages)
 
 
-def _render(rollouts: list[Rollout], config: EvalConfig, start: float) -> Group:
+def _render(
+    rollouts: list[Rollout], config: EvalConfig, start: float, pager: Pager
+) -> Group:
     now = time.time()
     warning = _warning(config)
     # `{warning}\n\n{overview}` — the caution sits at the very top, blank line, then the overview.
     header = Group(warning, Text(""), Overview(config)) if warning else Overview(config)
     # Measure the fixed top (header + progress + rule) so the rollout rows fill the rest of the
-    # screen; page through them on a timer when they'd overflow (rich would otherwise truncate).
+    # screen; page through them (timer, or the left/right arrows) when they'd overflow (rich would
+    # otherwise truncate).
     top = Group(header, Progress(rollouts, start), Rule(style="dim"))
     rows_per_page = max(1, _CONSOLE.size.height - len(_CONSOLE.render_lines(top)) - 1)
-    page_groups, index, count = _paginate(_groups(rollouts), rows_per_page, now)
+    page_groups, index, count = _paginate(_groups(rollouts), rows_per_page, pager, now)
     progress = Progress(rollouts, start, page=(index + 1, count) if count > 1 else None)
     return Group(
         header,
@@ -539,6 +572,10 @@ def _render(rollouts: list[Rollout], config: EvalConfig, start: float) -> Group:
 
 @contextlib.asynccontextmanager
 async def dashboard(rollouts: list[Rollout], config: EvalConfig, start: float):
-    """Refresh the live eval view until the `with` block exits, then a final frame."""
-    async with live_view(lambda: _render(rollouts, config, start)):
+    """Refresh the live eval view until the `with` block exits, then a final frame. Left/right
+    arrows page through rollout rows when they overflow the screen."""
+    pager = Pager()
+    async with live_view(
+        lambda: _render(rollouts, config, start, pager), on_key=pager.on_key
+    ):
         yield

verifiers-0.1.15.dev416/verifiers/v1/scoring.py

@@ -0,0 +1,166 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Sequence
+from decimal import Decimal, InvalidOperation
+from typing import TYPE_CHECKING
+
+from math_verify import parse, verify
+from math_verify.errors import TimeoutException as MathVerifyTimeout
+
+from verifiers.v1.errors import SandboxError
+
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes import Runtime
+    from verifiers.v1.trace import Trace
+
+BOXED_START = "\\boxed{"
+ANSI_RE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
+PYTEST_OUTCOMES = ("PASSED", "FAILED", "ERROR", "SKIPPED", "XFAIL", "XPASS")
+
+
+def extract_boxed_answer(text: str, strict: bool = False) -> str:
+    start = text.rfind(BOXED_START)
+    if start == -1:
+        return "" if strict else text
+
+    # Regex is the wrong tool for nested braces, so walk the final box by hand.
+    answer_start = start + len(BOXED_START)
+    depth = 1
+    for index, char in enumerate(text[answer_start:], start=answer_start):
+        if char == "{":
+            depth += 1
+        elif char == "}":
+            depth -= 1
+        if depth == 0:
+            return text[answer_start:index]
+
+    return "" if strict else text
+
+
+async def read_answer_file_or_last_reply(
+    runtime: "Runtime", path: str, trace: "Trace"
+) -> str:
+    try:
+        answer = (await runtime.read(path)).decode(errors="replace").strip()
+    except (FileNotFoundError, OSError, SandboxError):
+        answer = ""
+    return answer or trace.last_reply
+
+
+def parse_judge_choice(
+    content: str | None, choices: Sequence[str] = ("A", "B", "C")
+) -> str | None:
+    if not content or ("<think>" in content and "</think>" not in content):
+        return None
+
+    text = content.rsplit("</think>", 1)[-1].strip()
+    text = extract_boxed_answer(text, strict=True).strip() or text
+
+    text_upper = text.upper()
+    choices_by_upper = {choice.upper(): choice for choice in choices}
+    allowed = "|".join(re.escape(choice) for choice in choices_by_upper)
+    choice_re = rf"(?<!\w)({allowed})(?!\w)"
+    verdict_re = (
+        r"(?:^|\n)\s*(?:FINAL\s+JUDGMENT|FINAL\s+ANSWER|FINAL\s+VERDICT|"
+        r"JUDGMENT|VERDICT|ANSWER)\s*(?:IS\s*)?[:\-]?\s*"
+    )
+
+    verdict = re.search(verdict_re, text_upper)
+    if verdict:
+        match = re.search(choice_re, text_upper[verdict.end() :])
+        return choices_by_upper.get(match.group(1)) if match else None
+
+    matches = re.findall(choice_re, text_upper)
+    return choices_by_upper.get(matches[-1]) if matches else None
+
+
+def verify_boxed_math_answer(
+    response: str | None, answer: str, *, timeout_seconds: int = 5
+) -> float:
+    if not response or ("<think>" in response and "</think>" not in response):
+        return 0.0
+
+    prediction_text = response.rsplit("</think>", 1)[-1]
+    prediction = extract_boxed_answer(prediction_text, strict=True).strip()
+    gold = (extract_boxed_answer(answer, strict=True) or answer).strip()
+    if not prediction or not gold:
+        return 0.0
+
+    # math-verify expects both sides as boxed math expressions.
+    try:
+        parsed_gold = parse(f"\\boxed{{{gold}}}", parsing_timeout=timeout_seconds)
+        parsed_prediction = parse(
+            f"\\boxed{{{prediction}}}",
+            parsing_timeout=timeout_seconds,
+        )
+        return float(
+            verify(
+                parsed_gold,
+                parsed_prediction,
+                timeout_seconds=timeout_seconds,
+            )
+        )
+    except (Exception, MathVerifyTimeout):
+        return 0.0
+
+
+def compare_stdout_results(
+    exec_stdout: str, expected: str, *, tolerance: float = 1e-3
+) -> bool:
+    if exec_stdout.strip() == expected.strip():
+        return True
+
+    actual_lines = [line.strip() for line in exec_stdout.splitlines() if line.strip()]
+    expected_lines = [line.strip() for line in expected.splitlines() if line.strip()]
+    if actual_lines == expected_lines:
+        return True
+
+    actual_tokens = [token for line in actual_lines for token in line.split()]
+    expected_tokens = [token for line in expected_lines for token in line.split()]
+    if actual_tokens == expected_tokens:
+        return True
+    if len(actual_tokens) != len(expected_tokens) or not actual_tokens:
+        return False
+
+    # Code datasets often allow tiny floating-point drift in stdout.
+    try:
+        actual_numbers = [Decimal(token) for token in actual_tokens]
+        expected_numbers = [Decimal(token) for token in expected_tokens]
+    except (InvalidOperation, ValueError):
+        return False
+    if not all(number.is_finite() for number in actual_numbers + expected_numbers):
+        return False
+
+    limit = Decimal(str(tolerance))
+    return all(
+        abs(actual - expected) <= limit
+        for actual, expected in zip(actual_numbers, expected_numbers)
+    )
+
+
+def parse_pytest_outcomes(output: str | None) -> dict[str, str]:
+    outcomes: dict[str, str] = {}
+    for raw_line in ANSI_RE.sub("", output or "").splitlines():
+        parts = raw_line.strip().split(maxsplit=1)
+        if len(parts) != 2:
+            continue
+
+        outcome, test_id = parts
+        if outcome not in PYTEST_OUTCOMES:
+            continue
+
+        # SKIPPED short-summary rows use "[N] file.py:line: reason", not a node id.
+        if test_id.startswith("["):
+            continue
+
+        # These summary rows append " - <reason>"; passing node ids do not.
+        if outcome in ("FAILED", "ERROR", "XFAIL", "XPASS") and " - " in test_id:
+            parts = test_id.split(" - ")
+            for index in range(1, len(parts)):
+                node_id = " - ".join(parts[:index])
+                if node_id.count("[") == node_id.count("]"):
+                    test_id = node_id
+                    break
+        outcomes[test_id.rstrip()] = outcome
+    return outcomes

verifiers-0.1.15.dev414/verifiers/v1/cli/dashboard/base.py

@@ -1,32 +0,0 @@
-"""The shared dashboard engine: a rich `Live` view on a fixed refresh tick.
-
-The eval and validate dashboards each build their own frame; `live_view` just drives whichever
-`render` it's given — refreshing on a timer and drawing a final frame on exit.
-"""
-
-import asyncio
-import contextlib
-from collections.abc import Callable
-
-from rich.console import Group
-from rich.live import Live
-
-
-@contextlib.asynccontextmanager
-async def live_view(render: Callable[[], Group]):
-    """Refresh `render()` every 0.25s until the block exits, then draw one final frame."""
-    with Live(render(), auto_refresh=False) as live:
-
-        async def loop() -> None:
-            while True:
-                await asyncio.sleep(0.25)
-                live.update(render(), refresh=True)
-
-        task = asyncio.create_task(loop())
-        try:
-            yield
-        finally:
-            task.cancel()
-            with contextlib.suppress(asyncio.CancelledError):
-                await task
-            live.update(render(), refresh=True)