verifiers 0.1.15.dev443__tar.gz → 0.2.1.dev1__tar.gz

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev443
+Version: 0.2.1.dev1
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/tests/v1/test_e2e.py

@@ -74,6 +74,11 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns >= 2  # tool call + answer
     assert trace.reward == 1.0
+    # The interception server captured the advertised tools onto the trace (for tool-use SFT):
+    # the null harness offered the task's MCP tool as `echo_back`, schema included.
+    assert trace.tools is not None
+    (echo_tool,) = [t for t in trace.tools if t.name == "echo_back"]
+    assert "message" in echo_tool.parameters.get("properties", {})
 
 
 @pytest.mark.e2e

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/tests/v1/test_trace.py

@@ -60,6 +60,7 @@ def test_wire_trace_round_trip():
     # carry node `parent` links for `num_branches` to survive.
     tr = vf.Trace[MyTask, vf.State](
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="a")),
+        tools=[vf.Tool(name="echo", description="", parameters={"type": "object"})],
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
             MessageNode(parent=0, message=AssistantMessage(content="a1"), sampled=True),
@@ -80,6 +81,9 @@ def test_wire_trace_round_trip():
     assert rt.reward == 1.0  # property recomputed from `rewards`
     assert rt.stop_condition == "done"
     assert rt.info == {"build": "ok"}
+    assert (
+        rt.tools == tr.tools
+    )  # the advertised tools persist (tool-use SFT reads them)
     assert rt.task.data.model_extra == {
         "answer": "a"
     }  # taskset extras preserved on WireTaskData

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/verifiers/v1/dialects/chat.py

@@ -96,6 +96,9 @@ def parse_message(raw: dict) -> Message:
 
 
 def parse_tools(raw: list[dict] | None) -> list[Tool] | None:
+    # `or None` so a tools array with no function entries (e.g. only `custom`/built-in
+    # tools) parses to None, not [] — the same contract as the anthropic/responses
+    # dialects, and what keeps an empty parse from clearing `Trace.tools`.
     if not raw:
         return None
     return [
@@ -107,7 +110,7 @@ def parse_tools(raw: list[dict] | None) -> list[Tool] | None:
         )
         for t in raw
         if t.get("type", "function") == "function"
-    ]
+    ] or None
 
 
 # --- vf -> chat wire ----------------------------------------------------------

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/verifiers/v1/interception/server.py

@@ -43,7 +43,7 @@ from verifiers.v1.errors import (
     UserError,
 )
 from verifiers.v1.trace import Trace
-from verifiers.v1.types import Messages
+from verifiers.v1.types import Messages, Tool
 
 if TYPE_CHECKING:
     from verifiers.v1.mcp import Respond
@@ -259,7 +259,10 @@ class InterceptionServer:
         # dialect's typed view for building the trace; the renderer re-derives its own from `body`.
         # A user simulator extends both each turn (`dialect.extend` for wire, `prompt` for trace).
         prompt: Messages
-        prompt, _ = dialect.parse_request(body)
+        # `tools` is recorded onto the trace only when a turn commits (below / in `_stream`):
+        # the request is ground truth for what the model saw, but a refused or failed request
+        # was never seen at all.
+        prompt, tools = dialect.parse_request(body)
         # Cache the opening so retries do not advance the simulator twice.
         if (
             session.user is not None
@@ -273,7 +276,7 @@ class InterceptionServer:
             # If the simulator ended at the open (its task's `@stop` now fires), the loop's
             # `refused()` below halts the harness before any model call — no special-casing here.
         if dialect.streaming(body):
-            return await self._stream(request, session, dialect, body, prompt)
+            return await self._stream(request, session, dialect, body, prompt, tools)
         headers = request.headers.copy()
         # A user simulator turns one program request into a multi-turn exchange: after each
         # model turn the simulator's reply is injected as a user turn and the model is
@@ -356,6 +359,12 @@ class InterceptionServer:
             )
             turn.commit(response)  # one node per new message;
             # branches fall out of walking the graph (see Trace.branches / verifiers.v1.graph)
+            if tools:
+                # Persist the advertised tools now that the model actually saw them (see
+                # `Trace.tools`). Only a tools-bearing turn updates it (truthy, so an empty
+                # parse or a trailing tool-less call never erases the schemas the recorded
+                # tool calls need).
+                session.trace.tools = tools
             # Hand back to the program when the model wants a tool (the program runs it) or
             # when there's no user simulator to keep the conversation going.
             if response.message.tool_calls or session.user is None:
@@ -389,6 +398,7 @@ class InterceptionServer:
         dialect: Dialect,
         body: dict,
         prompt: Messages,
+        tools: list[Tool] | None = None,
     ) -> web.StreamResponse:
         """A streamed (SSE) model turn: relay the provider's stream through to the program,
         incrementally assembling the response to record on the trace. Single-shot — a streamed
@@ -502,6 +512,8 @@ class InterceptionServer:
             if parser_error is not None:
                 raise parser_error
             turn.commit(parser.finish())
+            if tools:  # record only on commit, like the non-streaming path
+                session.trace.tools = tools
             logger.debug("intercept stream turn: id=%s", session.trace.id)
         finally:
             # Release the withheld events only now — after the commit — then close.

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/verifiers/v1/legacy.py

@@ -20,6 +20,7 @@ from typing import Any
 
 import zmq
 import zmq.asyncio
+from pydantic import ValidationError
 
 from verifiers.v1.clients.config import ClientConfig, TrainClientConfig
 from verifiers.v1.serve.server import EnvServer
@@ -37,6 +38,7 @@ from verifiers.v1.types import (
     Response,
     SamplingConfig,
     SystemMessage,
+    Tool,
     ToolCall,
     ToolMessage,
     TurnTokens,
@@ -61,6 +63,22 @@ def _as_dict(obj: Any) -> Any:
     return obj
 
 
+def _to_v1_tools(raw: Any) -> list[Tool] | None:
+    """Map v0 ``RolloutOutput.tool_defs`` onto ``Trace.tools``. The v0 and v1 ``Tool``
+    shapes are identical (name/description/parameters/strict), so this is a re-validation;
+    malformed entries are dropped rather than failing the whole trace mapping."""
+    defs: list[Tool] = []
+    for t in raw or []:
+        t = _as_dict(t)
+        if not isinstance(t, dict):
+            continue
+        try:
+            defs.append(Tool.model_validate(t))
+        except ValidationError:
+            continue
+    return defs or None
+
+
 def _text(content: Any) -> str:
     """Flatten a message ``content`` (str, or a list of content parts) to text."""
     if isinstance(content, str):
@@ -234,6 +252,7 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
             type="Task",
             data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")),
         ),
+        tools=_to_v1_tools(out.get("tool_defs")),
         rewards={"reward": float(out.get("reward") or 0.0)},
         metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
         is_completed=bool(out.get("is_completed", True)),

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/verifiers/v1/push.py

@@ -56,6 +56,12 @@ def trace_to_sample(trace: Trace, rollout_number: int = 1) -> dict[str, Any]:
         "prompt": [],
         "completion": dump(branches[-1].messages) if branches else [],
         "answer": task.get("answer"),
+        # The tools advertised to the model (`Trace.tools`); keyed `tool_defs` because the
+        # v0 sample format already carries it there, so bridged and native rollouts render
+        # the same on the platform.
+        "tool_defs": [t.model_dump(mode="json", exclude_none=True) for t in trace.tools]
+        if trace.tools
+        else None,
         "reward": trace.reward,
         "timing": trace.timing.model_dump(mode="json", exclude_none=True),
         "is_completed": trace.is_completed,

{verifiers-0.1.15.dev443 → verifiers-0.2.1.dev1}/verifiers/v1/trace.py

@@ -24,6 +24,7 @@ from verifiers.v1.types import (
     AssistantMessage,
     Messages,
     StrictBaseModel,
+    Tool,
     ToolMessage,
     Usage,
     content_text,
@@ -200,6 +201,11 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
     """The runtime's full config plus its provisioned resource ID."""
     nodes: list[MessageNode] = Field(default_factory=list)
     """The message graph; branches are derived views and storage stays linear in turns."""
+    tools: list[Tool] | None = None
+    """The tools advertised to the model, recorded when an intercepted turn commits (last
+    committed turn wins) — never from a refused/failed request the model never saw. The full
+    advertised list (not just tools called), so tool-use SFT can re-render the exact prompt;
+    a trace-level snapshot: mid-rollout changes collapse to the last set the model saw."""
 
     rewards: dict[str, float] = Field(default_factory=dict)
     """Weighted contributions from task rewards, group rewards, and judges."""