verifiers 0.2.0__tar.gz → 0.2.1.dev2__tar.gz

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.0
+Version: 0.2.1.dev2
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/tests/v1/test_e2e.py

@@ -74,6 +74,11 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns >= 2  # tool call + answer
     assert trace.reward == 1.0
+    # The interception server captured the advertised tools onto the trace (for tool-use SFT):
+    # the null harness offered the task's MCP tool as `echo_back`, schema included.
+    assert trace.tools is not None
+    (echo_tool,) = [t for t in trace.tools if t.name == "echo_back"]
+    assert "message" in echo_tool.parameters.get("properties", {})
 
 
 @pytest.mark.e2e

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/tests/v1/test_trace.py

@@ -60,6 +60,7 @@ def test_wire_trace_round_trip():
     # carry node `parent` links for `num_branches` to survive.
     tr = vf.Trace[MyTask, vf.State](
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="a")),
+        tools=[vf.Tool(name="echo", description="", parameters={"type": "object"})],
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
             MessageNode(parent=0, message=AssistantMessage(content="a1"), sampled=True),
@@ -80,6 +81,9 @@ def test_wire_trace_round_trip():
     assert rt.reward == 1.0  # property recomputed from `rewards`
     assert rt.stop_condition == "done"
     assert rt.info == {"build": "ok"}
+    assert (
+        rt.tools == tr.tools
+    )  # the advertised tools persist (tool-use SFT reads them)
     assert rt.task.data.model_extra == {
         "answer": "a"
     }  # taskset extras preserved on WireTaskData

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/cli/dashboard/base.py

@@ -9,6 +9,8 @@ from collections.abc import Callable, Iterator
 from rich.console import Group
 from rich.live import Live
 
+from verifiers.v1.utils.interrupt import cleaning_up
+
 try:  # POSIX-only terminal control; absent on Windows, where key reading is skipped.
     import termios
     import tty
@@ -82,10 +84,17 @@ async def live_view(
     """Refresh `render()` every 0.25s until the block exits, then draw one final frame. When
     `on_key` is given, left/right arrow presses are dispatched to it (and redraw at once)."""
     with Live(render(), auto_refresh=False) as live:
+        stopping = False
 
         async def loop() -> None:
             while True:
-                await asyncio.sleep(0.25)
+                try:
+                    await asyncio.sleep(0.25)
+                except asyncio.CancelledError:
+                    # Ctrl-C cancels every task at once; keep refreshing so the cleanup notice
+                    # stays live through teardown, stopping only when live_view tears down.
+                    if stopping or not cleaning_up():
+                        raise
                 live.update(render(), refresh=True)
 
         task = asyncio.create_task(loop())
@@ -93,6 +102,7 @@ async def live_view(
             try:
                 yield
             finally:
+                stopping = True
                 task.cancel()
                 with contextlib.suppress(asyncio.CancelledError):
                     await task

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/cli/dashboard/eval.py

@@ -14,6 +14,7 @@ from rich.text import Text
 
 from verifiers.v1.cli.dashboard.base import live_view
 from verifiers.v1.cli.output import output_path
+from verifiers.v1.utils.interrupt import cleaning_up
 from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.rollout import Phase, Rollout
 from verifiers.v1.trace import Trace
@@ -110,6 +111,22 @@ def _aligned(rows: list[list[str]]) -> list[str]:
     ]
 
 
+def _interrupt_footer() -> Group | None:
+    """The graceful-shutdown notice under the rollouts once Ctrl-C has begun teardown — the
+    on-screen echo of the warning (console logging is silenced in rich mode); a further Ctrl-C
+    is ignored while it shows. Sits beside the `--push` line, mirroring its placement."""
+    if not cleaning_up():
+        return None
+    return Group(
+        Rule(style="dim"),
+        Text(
+            "interrupted — cleaning up, tearing down containers/sandboxes. "
+            "please wait; a further ctrl-c is ignored.",
+            style="yellow",
+        ),
+    )
+
+
 def _warning(config: EvalConfig) -> Text | None:
     """A local-runtime caution for a code-running harness (none for the tool-less `null`),
     shown above the overview rather than as a row in it."""
@@ -561,11 +578,11 @@ def _render(
     now = time.time()
     warning = _warning(config)
     header = Group(warning, Text(""), Overview(config)) if warning else Overview(config)
-    # The --push status line appears under the rollouts once the upload starts (None during the run
-    # / when off). Measure the fixed top (header + progress + rule) and the footer so the rollout
-    # rows fill what's left; page through them (timer / arrows) when they'd overflow (else rich
-    # truncates).
-    footer = _push_footer(push)
+    # The --push status line (and, on Ctrl-C, the cleanup notice) appear under the rollouts. Measure
+    # the fixed top (header + progress + rule) and the footer so the rollout rows fill what's left;
+    # page through them (timer / arrows) when they'd overflow (else rich truncates).
+    footers = [f for f in (_push_footer(push), _interrupt_footer()) if f is not None]
+    footer = Group(*footers) if footers else None
     top = Group(header, Progress(rollouts, start), Rule(style="dim"))
     reserved = len(_CONSOLE.render_lines(top))
     if footer is not None:

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/cli/eval/main.py

@@ -2,12 +2,12 @@
 
 import asyncio
 import logging
-import signal
 import sys
 
 from pydantic_config import cli
 
 import verifiers.v1 as vf
+from verifiers.v1.utils.interrupt import install as install_interrupt
 from verifiers.v1.utils.logging import setup_logging
 from verifiers.v1.cli.output import output_path, write_config
 from verifiers.v1.cli.resolve import (
@@ -80,21 +80,30 @@ def main(argv: list[str] | None = None) -> None:
         logging.lastResort = None
     else:
         setup_logging(level, log_file=log_file, console=True)
-    # Make SIGTERM behave like Ctrl-C (SIGINT) so a killed/timed-out eval still runs each
-    # rollout's `finally` (tears down containers/sandboxes) and any worker pool it spawned.
-    signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt()))
+    # First Ctrl-C / SIGTERM warns and raises KeyboardInterrupt so a killed/timed-out eval still
+    # runs each rollout's `finally` (tears down containers/sandboxes) and any worker pool it
+    # spawned; further signals during that cleanup are swallowed so an impatient second Ctrl-C
+    # can't orphan those resources.
+    install_interrupt()
 
-    if config.is_legacy:  # v0 backwards-compat: run the classic env, bridged to Traces
-        from verifiers.v1.legacy import run_legacy_eval
+    try:
+        if (
+            config.is_legacy
+        ):  # v0 backwards-compat: run the classic env, bridged to Traces
+            from verifiers.v1.legacy import run_legacy_eval
 
-        traces = asyncio.run(run_legacy_eval(config))
-    elif config.server:  # opt-in: drive rollouts through the env-server worker pool
-        from verifiers.v1.cli.eval.runner import run_eval_server
+            traces = asyncio.run(run_legacy_eval(config))
+        elif config.server:  # opt-in: drive rollouts through the env-server worker pool
+            from verifiers.v1.cli.eval.runner import run_eval_server
 
-        traces = asyncio.run(run_eval_server(config))
-    else:  # in-process (default), with or without the live dashboard
-        env = vf.Environment(config)
-        traces = asyncio.run(run_eval(env, config))
+            traces = asyncio.run(run_eval_server(config))
+        else:  # in-process (default), with or without the live dashboard
+            env = vf.Environment(config)
+            traces = asyncio.run(run_eval(env, config))
+    except KeyboardInterrupt:
+        # Graceful cleanup has already run (each rollout's `finally`); partial results are on
+        # disk. Exit on the conventional Ctrl-C code without a traceback.
+        raise SystemExit(130)
     if config.push and not rich:
         from verifiers.v1.push import push_traces
 

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/dialects/chat.py

@@ -96,6 +96,9 @@ def parse_message(raw: dict) -> Message:
 
 
 def parse_tools(raw: list[dict] | None) -> list[Tool] | None:
+    # `or None` so a tools array with no function entries (e.g. only `custom`/built-in
+    # tools) parses to None, not [] — the same contract as the anthropic/responses
+    # dialects, and what keeps an empty parse from clearing `Trace.tools`.
     if not raw:
         return None
     return [
@@ -107,7 +110,7 @@ def parse_tools(raw: list[dict] | None) -> list[Tool] | None:
         )
         for t in raw
         if t.get("type", "function") == "function"
-    ]
+    ] or None
 
 
 # --- vf -> chat wire ----------------------------------------------------------

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/interception/server.py

@@ -43,7 +43,7 @@ from verifiers.v1.errors import (
     UserError,
 )
 from verifiers.v1.trace import Trace
-from verifiers.v1.types import Messages
+from verifiers.v1.types import Messages, Tool
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import Respond
@@ -259,7 +259,10 @@ class InterceptionServer:
         # dialect's typed view for building the trace; the renderer re-derives its own from `body`.
         # A user simulator extends both each turn (`dialect.extend` for wire, `prompt` for trace).
         prompt: Messages
-        prompt, _ = dialect.parse_request(body)
+        # `tools` is recorded onto the trace only when a turn commits (below / in `_stream`):
+        # the request is ground truth for what the model saw, but a refused or failed request
+        # was never seen at all.
+        prompt, tools = dialect.parse_request(body)
         # Cache the opening so retries do not advance the simulator twice.
         if (
             session.user is not None
@@ -273,7 +276,7 @@ class InterceptionServer:
             # If the simulator ended at the open (its task's `@stop` now fires), the loop's
             # `refused()` below halts the harness before any model call — no special-casing here.
         if dialect.streaming(body):
-            return await self._stream(request, session, dialect, body, prompt)
+            return await self._stream(request, session, dialect, body, prompt, tools)
         headers = request.headers.copy()
         # A user simulator turns one program request into a multi-turn exchange: after each
         # model turn the simulator's reply is injected as a user turn and the model is
@@ -356,6 +359,12 @@ class InterceptionServer:
             )
             turn.commit(response)  # one node per new message;
             # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph)
+            if tools:
+                # Persist the advertised tools now that the model actually saw them (see
+                # `Trace.tools`). Only a tools-bearing turn updates it (truthy, so an empty
+                # parse or a trailing tool-less call never erases the schemas the recorded
+                # tool calls need).
+                session.trace.tools = tools
             # Hand back to the program when the model wants a tool (the program runs it) or
             # when there's no user simulator to keep the conversation going.
             if response.message.tool_calls or session.user is None:
@@ -389,6 +398,7 @@ class InterceptionServer:
         dialect: Dialect,
         body: dict,
         prompt: Messages,
+        tools: list[Tool] | None = None,
     ) -> web.StreamResponse:
         """A streamed (SSE) model turn: relay the provider's stream through to the program,
         incrementally assembling the response to record on the trace. Single-shot — a streamed
@@ -502,6 +512,8 @@ class InterceptionServer:
             if parser_error is not None:
                 raise parser_error
             turn.commit(parser.finish())
+            if tools:  # record only on commit, like the non-streaming path
+                session.trace.tools = tools
             logger.debug("intercept stream turn: id=%s", session.trace.id)
         finally:
             # Release the withheld events only now — after the commit — then close.

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/legacy.py

@@ -20,6 +20,7 @@ from typing import Any
 
 import zmq
 import zmq.asyncio
+from pydantic import ValidationError
 
 from verifiers.v1.clients.config import ClientConfig, TrainClientConfig
 from verifiers.v1.serve.server import EnvServer
@@ -37,6 +38,7 @@ from verifiers.v1.types import (
     Response,
     SamplingConfig,
     SystemMessage,
+    Tool,
     ToolCall,
     ToolMessage,
     TurnTokens,
@@ -61,6 +63,22 @@ def _as_dict(obj: Any) -> Any:
     return obj
 
 
+def _to_v1_tools(raw: Any) -> list[Tool] | None:
+    """Map v0 ``RolloutOutput.tool_defs`` onto ``Trace.tools``. The v0 and v1 ``Tool``
+    shapes are identical (name/description/parameters/strict), so this is a re-validation;
+    malformed entries are dropped rather than failing the whole trace mapping."""
+    defs: list[Tool] = []
+    for t in raw or []:
+        t = _as_dict(t)
+        if not isinstance(t, dict):
+            continue
+        try:
+            defs.append(Tool.model_validate(t))
+        except ValidationError:
+            continue
+    return defs or None
+
+
 def _text(content: Any) -> str:
     """Flatten a message ``content`` (str, or a list of content parts) to text."""
     if isinstance(content, str):
@@ -234,6 +252,7 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
             type="Task",
             data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")),
         ),
+        tools=_to_v1_tools(out.get("tool_defs")),
         rewards={"reward": float(out.get("reward") or 0.0)},
         metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
         is_completed=bool(out.get("is_completed", True)),

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/push.py

@@ -56,6 +56,12 @@ def trace_to_sample(trace: Trace, rollout_number: int = 1) -> dict[str, Any]:
         "prompt": [],
         "completion": dump(branches[-1].messages) if branches else [],
         "answer": task.get("answer"),
+        # The tools advertised to the model (`Trace.tools`); keyed `tool_defs` because the
+        # v0 sample format already carries it there, so bridged and native rollouts render
+        # the same on the platform.
+        "tool_defs": [t.model_dump(mode="json", exclude_none=True) for t in trace.tools]
+        if trace.tools
+        else None,
         "reward": trace.reward,
         "timing": trace.timing.model_dump(mode="json", exclude_none=True),
         "is_completed": trace.is_completed,

{verifiers-0.2.0 → verifiers-0.2.1.dev2}/verifiers/v1/trace.py

@@ -24,6 +24,7 @@ from verifiers.v1.types import (
     AssistantMessage,
     Messages,
     StrictBaseModel,
+    Tool,
     ToolMessage,
     Usage,
     content_text,
@@ -200,6 +201,11 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
     """The runtime's full config plus its provisioned resource ID."""
     nodes: list[MessageNode] = Field(default_factory=list)
     """The message graph; branches are derived views and storage stays linear in turns."""
+    tools: list[Tool] | None = None
+    """The tools advertised to the model, recorded when an intercepted turn commits (last
+    committed turn wins) — never from a refused/failed request the model never saw. The full
+    advertised list (not just tools called), so tool-use SFT can re-render the exact prompt;
+    a trace-level snapshot: mid-rollout changes collapse to the last set the model saw."""
 
     rewards: dict[str, float] = Field(default_factory=dict)
     """Weighted contributions from task rewards, group rewards, and judges."""

verifiers-0.2.1.dev2/verifiers/v1/utils/interrupt.py

@@ -0,0 +1,33 @@
+"""Graceful-shutdown signal handling for the eval CLI: the first Ctrl-C/SIGTERM warns and
+raises KeyboardInterrupt so asyncio unwinds each rollout's teardown `finally`; further signals
+are swallowed so a second Ctrl-C can't orphan containers/sandboxes mid-cleanup."""
+
+import logging
+import signal
+
+logger = logging.getLogger(__name__)
+
+_cleaning_up = False
+
+
+def cleaning_up() -> bool:
+    """Whether a shutdown signal has begun graceful cleanup (read by the dashboard)."""
+    return _cleaning_up
+
+
+def install() -> None:
+    """Route SIGINT/SIGTERM through the graceful-shutdown handler."""
+
+    def handle(*_) -> None:
+        global _cleaning_up
+        first, _cleaning_up = not _cleaning_up, True
+        logger.warning(
+            "interrupted — cleaning up, please wait..."
+            if first
+            else "cleanup in progress — please wait (ctrl-c ignored)"
+        )
+        if first:
+            raise KeyboardInterrupt
+
+    signal.signal(signal.SIGINT, handle)
+    signal.signal(signal.SIGTERM, handle)