verifiers 0.3.1.dev39__tar.gz → 0.3.1.dev41__tar.gz

{verifiers-0.3.1.dev39 → verifiers-0.3.1.dev41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev39
+Version: 0.3.1.dev41
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev39 → verifiers-0.3.1.dev41}/tests/v1/fixtures/counter_tool_v1.py

@@ -44,9 +44,9 @@ class CounterTaskset(vf.Taskset[CounterTask, CounterConfig]):
                 vf.TaskData(
                     idx=0,
                     prompt=(
-                        f"Call the `counter_bump` tool {TARGET} times, one call per turn — wait "
-                        "for each result before the next. After the last result, reply with "
-                        "<answer>done</answer>."
+                        f"Call the `bump` tool from the `counter` MCP server {TARGET} times, "
+                        "one call per turn — wait for each result before the next. After the "
+                        "last result, reply with <answer>done</answer>."
                     ),
                 ),
                 self.config.task,

{verifiers-0.3.1.dev39 → verifiers-0.3.1.dev41}/tests/v1/fixtures/echo_acp_resume_v1.py

@@ -59,8 +59,9 @@ class ACPResumeEnv(vf.SingleAgentEnv):
             if not first.terminated:
                 segments.append(
                     await interaction.turn(
-                        "Call `resume_recall` with the codeword from my previous "
-                        "message, then reply with exactly the tool result."
+                        "Call the `recall` tool from the `resume` MCP server with the "
+                        "codeword from my previous message, then reply with exactly the "
+                        "tool result."
                     )
                 )
             interaction.trace.info["acp_segments"] = [

{verifiers-0.3.1.dev39 → verifiers-0.3.1.dev41}/tests/v1/fixtures/echo_tool_v1.py

@@ -18,7 +18,7 @@ ECHO_TOKEN = "ok-7f3"  # the tool stamps this; only a real tool call can surface
 
 
 class EchoToolset(vf.Toolset[vf.ToolsetConfig]):
-    TOOL_PREFIX = "echo"  # the model sees `echo_back` (matches the prompt)
+    TOOL_PREFIX = "echo"
 
     @vf.tool
     def back(self, message: str) -> str:
@@ -53,7 +53,8 @@ class EchoToolTaskset(vf.Taskset[EchoToolTask, EchoToolConfig]):
                 vf.TaskData(
                     idx=0,
                     prompt=(
-                        f'Call the `echo_back` tool with the message "{PHRASE}", then reply '
+                        f"Call the `back` tool from the `echo` MCP server with the message "
+                        f'"{PHRASE}", then reply '
                         "with exactly what it returns inside <answer></answer> tags."
                     ),
                 ),

{verifiers-0.3.1.dev39 → verifiers-0.3.1.dev41}/tests/v1/fixtures/tool_response_image_v1.py

@@ -13,7 +13,7 @@ SYSTEM = "Call the requested tool before answering."
 
 
 class VisionToolset(vf.Toolset[vf.ToolsetConfig]):
-    TOOL_PREFIX = "vision"  # the model sees `vision_snapshot`
+    TOOL_PREFIX = "vision"
 
     @vf.tool
     def snapshot(self) -> list:
@@ -60,8 +60,8 @@ class ToolResponseImageTaskset(
                 vf.TaskData(
                     idx=0,
                     prompt=(
-                        "Call the `vision_snapshot` tool exactly once. After it returns, "
-                        "reply with exactly `done`."
+                        "Call the `snapshot` tool from the `vision` MCP server exactly once. "
+                        "After it returns, reply with exactly `done`."
                     ),
                     system_prompt=SYSTEM,
                 ),

{verifiers-0.3.1.dev39 → verifiers-0.3.1.dev41}/tests/v1/test_e2e.py

@@ -288,10 +288,10 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     assert trace.ok
     assert trace.num_turns >= 2  # tool call + answer
     assert trace.reward == 1.0
-    # The interception server captured the advertised tools onto the trace (for tool-use SFT):
-    # the null harness offered the task's MCP tool as `echo_back`, schema included.
+    # The interception server captured the harness's advertised name and schema. Harnesses
+    # may qualify the same raw MCP tool differently, so the test checks its stable suffix.
     assert trace.tools
-    (echo_tool,) = [t for t in trace.tools if t.name == "echo_back"]
+    (echo_tool,) = [tool for tool in trace.tools if tool.name.endswith("back")]
     assert "message" in echo_tool.parameters.get("properties", {})
 
 
@@ -599,9 +599,9 @@ async def test_env_id_user_sim_with_tools(run_v1, tmp_path):
     assert assistant.task.data.prompt is None  # the scenario stayed off the wire
     assert user.num_turns >= 1  # the modeled user actually drove the exchange
     assert assistant.rewards["echoed"].score == 1.0  # the tool ran, mid-conversation
-    # The tool was advertised to the masked chat exactly as to any run.
+    # The tool was advertised to the masked chat, regardless of harness qualification.
     assert assistant.tools
-    assert any(tool.name == "echo_back" for tool in assistant.tools)
+    assert any(tool.name.endswith("back") for tool in assistant.tools)
 
 
 @pytest.mark.e2e

{verifiers-0.3.1.dev39 → verifiers-0.3.1.dev41}/verifiers/v1/acp/runner.py

@@ -1,6 +1,6 @@
 # /// script
 # requires-python = ">=3.10,<3.15"
-# dependencies = ["agent-client-protocol==0.11.0"]
+# dependencies = ["agent-client-protocol==0.12.1"]
 # ///
 """Run harness segments through an ACP agent."""
 
@@ -35,7 +35,6 @@ from acp.schema import (
 )
 
 MAX_PACKET_BYTES = 128 * 1024 * 1024
-LATE_UPDATE_GRACE_SECONDS = 1.0
 
 
 class VerifiersACPClient(Client):
@@ -43,7 +42,6 @@ class VerifiersACPClient(Client):
         self.visible_reply = ""
         self.message_id: str | None = None
         self.tool_calls: dict[str, str] = {}
-        self.output_changed = asyncio.Condition()
 
     def reset(self) -> None:
         self.visible_reply = ""
@@ -51,23 +49,19 @@ class VerifiersACPClient(Client):
         self.tool_calls = {}
 
     async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None:
-        async with self.output_changed:
-            if isinstance(update, ToolCall):
-                self.tool_calls[update.tool_call_id] = update.status or "pending"
-            elif isinstance(update, ToolCallUpdate):
-                if update.status:
-                    self.tool_calls[update.tool_call_id] = update.status
-            elif isinstance(update, AgentMessageChunk) and isinstance(
-                update.content, TextContentBlock
-            ):
-                message_id = getattr(update, "message_id", None)
-                if message_id is not None and message_id != self.message_id:
-                    self.visible_reply = ""
-                    self.message_id = message_id
-                self.visible_reply += update.content.text
-            else:
-                return
-            self.output_changed.notify_all()
+        if isinstance(update, ToolCall):
+            self.tool_calls[update.tool_call_id] = update.status or "pending"
+        elif isinstance(update, ToolCallUpdate):
+            if update.status:
+                self.tool_calls[update.tool_call_id] = update.status
+        elif isinstance(update, AgentMessageChunk) and isinstance(
+            update.content, TextContentBlock
+        ):
+            message_id = getattr(update, "message_id", None)
+            if message_id is not None and message_id != self.message_id:
+                self.visible_reply = ""
+                self.message_id = message_id
+            self.visible_reply += update.content.text
 
     async def request_permission(
         self,
@@ -148,24 +142,6 @@ async def prompt(
         detail = error.data.get("details") if isinstance(error.data, dict) else None
         raise RuntimeError(detail or str(error)) from error
 
-    # ACP 0.11 dispatches notifications in background tasks but resolves a request
-    # response directly in its receive loop. An agent that sends its final
-    # session/update immediately before session/prompt returns can therefore wake
-    # this coroutine before the update handler has run. Wait specifically for text:
-    # a completed tool update may also arrive first and must not hide a later reply.
-    def has_visible_reply() -> bool:
-        return bool(client.visible_reply.strip())
-
-    if not has_visible_reply():
-        async with client.output_changed:
-            try:
-                await asyncio.wait_for(
-                    client.output_changed.wait_for(has_visible_reply),
-                    timeout=LATE_UPDATE_GRACE_SECONDS,
-                )
-            except asyncio.TimeoutError:  # noqa: UP041 - Python 3.10 compatibility
-                pass
-
     tool_statuses = list(client.tool_calls.values())
     completed_tool_turn = (
         config.get("allow_empty_tool_reply", False)
@@ -173,7 +149,7 @@ async def prompt(
         and bool(tool_statuses)
         and all(status in ("completed", "failed") for status in tool_statuses)
     )
-    if not has_visible_reply() and not completed_tool_turn:
+    if not client.visible_reply.strip() and not completed_tool_turn:
         raise RuntimeError(
             "ACP agent produced no visible reply "
             f"(stop_reason={response.stop_reason}, tool_statuses={tool_statuses})"