verifiers 0.1.15.dev413__tar.gz → 0.1.15.dev415__tar.gz

{verifiers-0.1.15.dev413 → verifiers-0.1.15.dev415}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev413
+Version: 0.1.15.dev415
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.1.15.dev415/verifiers/v1/cli/dashboard/base.py

@@ -0,0 +1,108 @@
+"""The shared dashboard engine: a rich `Live` view on a fixed refresh tick.
+
+The eval and validate dashboards each build their own frame; `live_view` just drives whichever
+`render` it's given — refreshing on a timer and drawing a final frame on exit. When given an
+`on_key` handler it also reads left/right arrow presses from the terminal (see `_key_reader`).
+"""
+
+import asyncio
+import contextlib
+import os
+import sys
+from collections.abc import Callable, Iterator
+
+from rich.console import Group
+from rich.live import Live
+
+try:  # POSIX-only terminal control; absent on Windows, where key reading is skipped.
+    import termios
+    import tty
+
+    _HAS_TTY = True
+except ImportError:
+    _HAS_TTY = False
+
+# Arrow-key escape sequences → direction, covering both the normal (`ESC [`) and application
+# (`ESC O`) cursor-key modes a terminal might send.
+_ARROWS = {
+    b"\x1b[C": "right",
+    b"\x1bOC": "right",
+    b"\x1b[D": "left",
+    b"\x1bOD": "left",
+}
+
+
+@contextlib.contextmanager
+def _key_reader(
+    on_key: Callable[[str], None] | None, refresh: Callable[[], None]
+) -> Iterator[None]:
+    """Dispatch left/right arrow presses to `on_key`, redrawing immediately after each so the
+    view feels instant rather than waiting for the next refresh tick. A no-op unless `on_key` is
+    given and stdin is a real terminal. Puts stdin in cbreak mode (char-at-a-time, no echo, Ctrl+C
+    still interrupts) and reads it via the event loop — no extra thread — restoring the terminal on
+    exit."""
+    if on_key is None or not _HAS_TTY or not sys.stdin.isatty():
+        yield
+        return
+    fd = sys.stdin.fileno()
+    loop = asyncio.get_running_loop()
+    old = termios.tcgetattr(fd)
+    # carries a partial escape across reads — cbreak can split a key's 3 bytes over callbacks
+    buf = bytearray()
+
+    def _read() -> None:
+        try:
+            data = os.read(fd, 32)
+        except OSError:
+            return
+        if not data:
+            return
+        buf.extend(data)
+        fired = False
+        i = 0
+        while i < len(buf):  # a chunk may hold several presses; scan the whole buffer
+            if key := _ARROWS.get(bytes(buf[i : i + 3])):
+                on_key(key)
+                fired = True
+                i += 3
+            elif buf[i] == 0x1B and len(buf) - i < 3:
+                break  # incomplete escape at the tail — hold it for the next read
+            else:
+                i += 1
+        del buf[:i]
+        if fired:
+            refresh()
+
+    # setcbreak inside the try so the finally always restores the terminal, even if it (or
+    # add_reader) raises — otherwise stdin could be left in cbreak with echo off after we exit.
+    try:
+        tty.setcbreak(fd)
+        loop.add_reader(fd, _read)
+        yield
+    finally:
+        loop.remove_reader(fd)
+        termios.tcsetattr(fd, termios.TCSADRAIN, old)
+
+
+@contextlib.asynccontextmanager
+async def live_view(
+    render: Callable[[], Group], on_key: Callable[[str], None] | None = None
+):
+    """Refresh `render()` every 0.25s until the block exits, then draw one final frame. When
+    `on_key` is given, left/right arrow presses are dispatched to it (and redraw at once)."""
+    with Live(render(), auto_refresh=False) as live:
+
+        async def loop() -> None:
+            while True:
+                await asyncio.sleep(0.25)
+                live.update(render(), refresh=True)
+
+        task = asyncio.create_task(loop())
+        with _key_reader(on_key, lambda: live.update(render(), refresh=True)):
+            try:
+                yield
+            finally:
+                task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await task
+                live.update(render(), refresh=True)

{verifiers-0.1.15.dev413 → verifiers-0.1.15.dev415}/verifiers/v1/cli/dashboard/eval.py

@@ -1,13 +1,14 @@
 """The eval `--rich` dashboard: a config overview, a progress bar, and one line per rollout.
 
 Reads each `Rollout.trace`/`phase` every tick — no extra plumbing. Each row carries a bracketed
-phase marker that reads at a glance — `[setup]` (yellow), `[rollout]` (cyan), `[finalize]`
-(magenta), `[scoring]` (blue), `[success]` (green), `[error]` (red) — padded so the brackets
-line up in a column down the left edge. The reward shows only once a rollout is fully scored
-(phase DONE), so it never flips as scoring lands. A task's rollouts are grouped adjacently and joined
-by a left brace (╭│╰), so an episode (a task's n rollouts) reads as a unit. Every started
-rollout stays on screen (finished ones keep their result); the overview + progress sit on top,
-above a rule.
+phase marker that reads at a glance — `[pending]` (dim), `[setup]` (yellow), `[rollout]` (cyan),
+`[finalize]` (magenta), `[scoring]` (blue), `[success]` (green), `[error]` (red) — padded so the
+brackets line up in a column down the left edge. The reward shows only once a rollout is fully
+scored (phase DONE), so it never flips as scoring lands. A task's rollouts are grouped adjacently
+and joined by a left brace (╭│╰), so an episode (a task's n rollouts) reads as a unit. Every
+rollout is on screen from the start: one still queued behind the concurrency cap reads `[pending]`
+(its task is all that's known yet) until it begins, and finished ones keep their result. The
+overview + progress sit on top, above a rule.
 """
 
 import contextlib
@@ -44,6 +45,7 @@ _PAGE_SECONDS = 5.0  # rotate to the next page of rollouts this often when they
 _LABEL_WIDTH = len("timeouts")
 
 _STYLE = {
+    "pending": "dim",
     "setup": "yellow",
     "running": "cyan",
     "finalize": "magenta",
@@ -52,6 +54,7 @@ _STYLE = {
     "error": "red",
 }
 _MARK_LABEL = {
+    "pending": "pending",
     "setup": "setup",
     "running": "rollout",
     "finalize": "finalize",
@@ -217,8 +220,8 @@ def Progress(
         f"{len(done)}/{len(rollouts)} · {format_time(time.time() - start)} · "
         f"reward {reward} · err {err}"
     )
-    if page is not None:  # rollouts overflow the screen — show which page is on screen
-        stats += f"  (page {page[0]}/{page[1]})"
+    if page is not None:  # overflowing — show which page, and that the arrows page
+        stats += f"  (page {page[0]}/{page[1]} ◄ ►)"
     row = Table.grid(expand=True, padding=(0, 1))
     row.add_column(ratio=1)  # bar stretches to fill the width left of the stats
     row.add_column(justify="right", no_wrap=True)
@@ -357,17 +360,26 @@ def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
     return prompt, completion, cached, reasoning, nbranches
 
 
+def _started(rollout: Rollout) -> float:
+    # Sort key: when a rollout began (its setup start). A still-pending rollout has no trace
+    # yet, so it sorts last (+inf) — behind everything already in flight, in task order.
+    return (
+        rollout.trace.timing.setup.start if rollout.trace is not None else float("inf")
+    )
+
+
 def _groups(rollouts: list[Rollout]) -> list[list[Rollout]]:
     # The n rollouts of each task, grouped together (so they sit adjacent); groups ordered by
-    # earliest start, rollouts within a group by start. Finished ones stay (never removed).
+    # earliest start, rollouts within a group by start. Every rollout carries its `task` from
+    # construction, so ones still queued behind the concurrency cap (no trace yet) are grouped
+    # and shown too — as `[pending]`. Finished ones stay (never removed).
     by_task: dict[int, list[Rollout]] = {}
     for rollout in rollouts:
-        if rollout.trace is not None:
-            by_task.setdefault(rollout.trace.task.idx, []).append(rollout)
+        by_task.setdefault(rollout.task.idx, []).append(rollout)
     groups = list(by_task.values())
     for group in groups:
-        group.sort(key=lambda r: r.trace.timing.setup.start)
-    groups.sort(key=lambda g: g[0].trace.timing.setup.start)
+        group.sort(key=_started)
+    groups.sort(key=lambda g: _started(g[0]))
     return groups
 
 
@@ -385,6 +397,21 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
     for group in groups:
         for i, rollout in enumerate(group):
             t = rollout.trace
+            task = rollout.task
+            label = f"name={task.name[:32]}" if task.name else f"idx={task.idx}"
+            if (
+                t is None
+            ):  # queued behind the concurrency cap — only its task is known yet
+                rows.append(
+                    (
+                        _brace(i, len(group)),
+                        "pending",
+                        [f"task {label}", *[""] * 7],
+                        "",
+                        "",
+                    )
+                )
+                continue
             if rollout.phase == Phase.DONE:  # fully scored — reward is final
                 state = "error" if t.has_error else "success"
                 result = t.error.type if t.has_error else f"reward={t.reward:.2f}"
@@ -398,7 +425,6 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
                         stop = f"{stop} (truncated)".strip()
             else:
                 state, result, stop = rollout.phase, "", ""
-            label = f"name={t.task.name[:32]}" if t.task.name else f"idx={t.task.idx}"
             descriptor = (
                 rollout.runtime.descriptor if rollout.runtime is not None else None
             )
@@ -469,13 +495,42 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
     return grid
 
 
+class Pager:
+    """Which page of overflowing rollout rows is on screen. Auto-advances on a timer until the
+    user takes over with the left/right arrows, after which it stays where they leave it. `count`
+    (the page count, set each render by `_paginate`) gates the arrows: they're inert while a single
+    page fits, so a stray press before rollouts overflow can't switch off auto-advance or offset the
+    starting page once paging begins. The chosen page is clamped to `count` (it can shrink on a
+    resize; it otherwise only grows, as rollouts are never removed)."""
+
+    def __init__(self) -> None:
+        self.page = 0
+        self.manual = False
+        self.count = 1
+
+    def on_key(self, key: str) -> None:
+        if key in ("left", "right") and self.count > 1:
+            self.manual = True
+            self.page += 1 if key == "right" else -1
+
+    def index(self, now: float) -> int:
+        # Track the auto page while it drives, so the first arrow continues from what's on screen
+        # rather than jumping back to page 1. Clamp in manual mode (count can shrink on resize).
+        if not self.manual:
+            self.page = int(now / _PAGE_SECONDS) % self.count
+        else:
+            self.page = max(0, min(self.page, self.count - 1))
+        return self.page
+
+
 def _paginate(
-    groups: list[list[Rollout]], rows_per_page: int, now: float
+    groups: list[list[Rollout]], rows_per_page: int, pager: Pager, now: float
 ) -> tuple[list[list[Rollout]], int, int]:
     """Pack groups (a task's rollouts kept together) into pages of at most `rows_per_page` rows,
-    cycling to the next page every `_PAGE_SECONDS`. Returns (this page's groups, 0-based index,
-    page count) — a single page when everything already fits."""
+    selecting the one `pager` points at. Returns (this page's groups, 0-based index, page count) —
+    a single page when everything already fits."""
     if sum(len(g) for g in groups) <= rows_per_page:
+        pager.count = 1  # everything fits on one page — arrows stay inert
         return groups, 0, 1
     pages: list[list[list[Rollout]]] = []
     current: list[list[Rollout]] = []
@@ -488,20 +543,24 @@ def _paginate(
         used += len(group)
     if current:
         pages.append(current)
-    index = int(now / _PAGE_SECONDS) % len(pages)
+    pager.count = len(pages)
+    index = pager.index(now)
     return pages[index], index, len(pages)
 
 
-def _render(rollouts: list[Rollout], config: EvalConfig, start: float) -> Group:
+def _render(
+    rollouts: list[Rollout], config: EvalConfig, start: float, pager: Pager
+) -> Group:
     now = time.time()
     warning = _warning(config)
     # `{warning}\n\n{overview}` — the caution sits at the very top, blank line, then the overview.
     header = Group(warning, Text(""), Overview(config)) if warning else Overview(config)
     # Measure the fixed top (header + progress + rule) so the rollout rows fill the rest of the
-    # screen; page through them on a timer when they'd overflow (rich would otherwise truncate).
+    # screen; page through them (timer, or the left/right arrows) when they'd overflow (rich would
+    # otherwise truncate).
     top = Group(header, Progress(rollouts, start), Rule(style="dim"))
     rows_per_page = max(1, _CONSOLE.size.height - len(_CONSOLE.render_lines(top)) - 1)
-    page_groups, index, count = _paginate(_groups(rollouts), rows_per_page, now)
+    page_groups, index, count = _paginate(_groups(rollouts), rows_per_page, pager, now)
     progress = Progress(rollouts, start, page=(index + 1, count) if count > 1 else None)
     return Group(
         header,
@@ -513,6 +572,10 @@ def _render(rollouts: list[Rollout], config: EvalConfig, start: float) -> Group:
 
 @contextlib.asynccontextmanager
 async def dashboard(rollouts: list[Rollout], config: EvalConfig, start: float):
-    """Refresh the live eval view until the `with` block exits, then a final frame."""
-    async with live_view(lambda: _render(rollouts, config, start)):
+    """Refresh the live eval view until the `with` block exits, then a final frame. Left/right
+    arrows page through rollout rows when they overflow the screen."""
+    pager = Pager()
+    async with live_view(
+        lambda: _render(rollouts, config, start, pager), on_key=pager.on_key
+    ):
         yield

{verifiers-0.1.15.dev413 → verifiers-0.1.15.dev415}/verifiers/v1/rollout.py

@@ -51,9 +51,11 @@ logger = logging.getLogger(__name__)
 
 
 class Phase(StrEnum):
-    """A rollout's lifecycle phase (for display): provisioning, the harness driving,
-    post-run finalize, per-rollout + group scoring, then fully scored."""
+    """A rollout's lifecycle phase (for display): queued behind the concurrency cap,
+    provisioning, the harness driving, post-run finalize, per-rollout + group scoring,
+    then fully scored."""
 
+    PENDING = "pending"
     SETUP = "setup"
     RUNNING = "running"
     FINALIZE = "finalize"
@@ -93,9 +95,11 @@ class Rollout:
         `Environment.serving` context — so a rollout always has them and no runner has to thread
         them in."""
         self.interception = interception
-        self.phase = Phase.SETUP
-        """Lifecycle phase for display (see `Phase`); advanced through the rollout, and
-        set to DONE by the Episode once group scoring has run."""
+        self.phase = Phase.PENDING
+        """Lifecycle phase for display (see `Phase`); starts PENDING (queued behind the
+        concurrency cap) so the --rich dashboard can list it before it begins, advances to
+        SETUP the moment `run()` starts, and is set to DONE by the Episode once group
+        scoring has run."""
         self.runtime: Runtime | None = None
         """The runtime, set the moment `run()` creates it (so it's always tearable-down
         even if setup crashes) and torn down in `run()`'s `finally`; the --rich dashboard
@@ -139,6 +143,7 @@ class Rollout:
         `self.shared_urls` / `self.interception`)."""
         trace: Trace = Trace(task=self.task, state=state_cls(type(self.taskset))())
         self.trace = trace  # expose for the --rich dashboard
+        self.phase = Phase.SETUP  # leaving the queue: provisioning starts now
         trace.timing.setup.start = time.time()
         self.runtime = make_runtime(
             self.runtime_config, name=trace.id

verifiers-0.1.15.dev413/verifiers/v1/cli/dashboard/base.py

@@ -1,32 +0,0 @@
-"""The shared dashboard engine: a rich `Live` view on a fixed refresh tick.
-
-The eval and validate dashboards each build their own frame; `live_view` just drives whichever
-`render` it's given — refreshing on a timer and drawing a final frame on exit.
-"""
-
-import asyncio
-import contextlib
-from collections.abc import Callable
-
-from rich.console import Group
-from rich.live import Live
-
-
-@contextlib.asynccontextmanager
-async def live_view(render: Callable[[], Group]):
-    """Refresh `render()` every 0.25s until the block exits, then draw one final frame."""
-    with Live(render(), auto_refresh=False) as live:
-
-        async def loop() -> None:
-            while True:
-                await asyncio.sleep(0.25)
-                live.update(render(), refresh=True)
-
-        task = asyncio.create_task(loop())
-        try:
-            yield
-        finally:
-            task.cancel()
-            with contextlib.suppress(asyncio.CancelledError):
-                await task
-            live.update(render(), refresh=True)