verifiers 0.3.1.dev53__tar.gz → 0.3.1.dev55__tar.gz

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.1.dev53
+Version: 0.3.1.dev55
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/acp/__init__.py

@@ -41,7 +41,6 @@ class ACPConfig:
     mcp_urls: dict[str, str] | None = None
     system_prompt: str | None = None
     session_meta: JsonObject | None = None
-    allow_empty_tool_reply: bool = False
 
 
 class ACPHarness(Harness[ConfigT]):
@@ -215,7 +214,6 @@ class ACPHarnessSession(HarnessSession):
             "mcp_urls": self.mcp_urls,
             "system_prompt": self.config.system_prompt or "",
             "session_meta": self.config.session_meta or {},
-            "allow_empty_tool_reply": self.config.allow_empty_tool_reply,
         }
         async with self._lock:
             if self._closed:

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/acp/runner.py

@@ -30,8 +30,6 @@ from acp.schema import (
     PermissionOption,
     RequestPermissionResponse,
     TextContentBlock,
-    ToolCall,
-    ToolCallUpdate,
 )
 
 MAX_PACKET_BYTES = 128 * 1024 * 1024
@@ -41,20 +39,13 @@ class VerifiersACPClient(Client):
     def __init__(self) -> None:
         self.visible_reply = ""
         self.message_id: str | None = None
-        self.tool_calls: dict[str, str] = {}
 
     def reset(self) -> None:
         self.visible_reply = ""
         self.message_id = None
-        self.tool_calls = {}
 
     async def session_update(self, session_id: str, update: Any, **kwargs: Any) -> None:
-        if isinstance(update, ToolCall):
-            self.tool_calls[update.tool_call_id] = update.status or "pending"
-        elif isinstance(update, ToolCallUpdate):
-            if update.status:
-                self.tool_calls[update.tool_call_id] = update.status
-        elif isinstance(update, AgentMessageChunk) and isinstance(
+        if isinstance(update, AgentMessageChunk) and isinstance(
             update.content, TextContentBlock
         ):
             message_id = getattr(update, "message_id", None)
@@ -137,23 +128,10 @@ async def prompt(
         raise ValueError("ACP prompt has no content")
     client.reset()
     try:
-        response = await connection.prompt(session_id=session_id, prompt=blocks)
+        await connection.prompt(session_id=session_id, prompt=blocks)
     except RequestError as error:
         detail = error.data.get("details") if isinstance(error.data, dict) else None
         raise RuntimeError(detail or str(error)) from error
-
-    tool_statuses = list(client.tool_calls.values())
-    completed_tool_turn = (
-        config.get("allow_empty_tool_reply", False)
-        and response.stop_reason == "end_turn"
-        and bool(tool_statuses)
-        and all(status in ("completed", "failed") for status in tool_statuses)
-    )
-    if not client.visible_reply.strip() and not completed_tool_turn:
-        raise RuntimeError(
-            "ACP agent produced no visible reply "
-            f"(stop_reason={response.stop_reason}, tool_statuses={tool_statuses})"
-        )
     return client.visible_reply
 
 

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/harnesses/openclaw/harness.py

@@ -229,8 +229,6 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
             prompt=prompt,
             mcp_urls={},
             system_prompt=system_prompt,
-            # OpenClaw can end after its final tool completes without a text message.
-            allow_empty_tool_reply=True,
         )
 
     async def cleanup(self, trace: Trace, runtime: Runtime) -> None:

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/harnesses/pi/harness.py

@@ -200,6 +200,4 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
             prompt=prompt,
             # Pi's extension owns the task-scoped MCP configuration.
             mcp_urls={},
-            # Pi can end after its final tool completes without a text message.
-            allow_empty_tool_reply=True,
         )

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/runtimes/prime.py

@@ -31,6 +31,7 @@ from verifiers.v1.runtimes.base import (
     parse_gpu,
 )
 from verifiers.v1.runtimes.limiters import creation_limiter
+from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.prime import ensure_prime_auth
 
 logger = logging.getLogger(__name__)
@@ -183,18 +184,27 @@ class PrimeRuntime(Runtime):
                 )
                 or contextlib.nullcontext()
             ):
-                sandbox = await self._client.create(
-                    CreateSandboxRequest(
-                        name=self.name,
-                        labels=list(dict.fromkeys([*BASE_LABELS, *self.config.labels])),
-                        docker_image=self.config.image,
-                        vm=self.config.vm,
-                        guaranteed=self.config.guaranteed,
-                        environment_vars=self.env,
-                        **{k: v for k, v in options.items() if v is not None},
+                # Shielded through the id capture: a cancel that aborts the POST
+                # mid-flight leaves the platform creating a sandbox this side
+                # never learned the id of — teardown() then cannot delete it
+                async def create_and_capture_id():
+                    sandbox = await self._client.create(
+                        CreateSandboxRequest(
+                            name=self.name,
+                            labels=list(
+                                dict.fromkeys([*BASE_LABELS, *self.config.labels])
+                            ),
+                            docker_image=self.config.image,
+                            vm=self.config.vm,
+                            guaranteed=self.config.guaranteed,
+                            environment_vars=self.env,
+                            **{k: v for k, v in options.items() if v is not None},
+                        )
                     )
-                )
-            self.info.id = sandbox.id
+                    self.info.id = sandbox.id
+                    return sandbox
+
+                sandbox = await run_shielded(create_and_capture_id())
             # The create response says whether the platform already has the image:
             # `pending_image_build_id` set means a first-use auto-build is running and the
             # sandbox stays PENDING until it finishes (`wait_for_creation` gives that phase

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/serve/client.py

@@ -24,6 +24,7 @@ from verifiers.v1.episode import WireEpisode
 from verifiers.v1.serve.types import (
     BaseRequest,
     BaseResponse,
+    CancelRequest,
     HealthRequest,
     HealthResponse,
     RunRequest,
@@ -46,6 +47,10 @@ class EnvClient:
         self.socket.setsockopt(zmq.RCVHWM, 0)
         self.socket.connect(address)
         self._pending: dict[str, asyncio.Future[bytes]] = {}
+        # Strong refs to in-flight fire-and-forget cancels: the loop only
+        # holds weak references to tasks, so an unreferenced one can be
+        # garbage-collected before it ever sends
+        self._cancel_tasks: set[asyncio.Task] = set()
         self._receiver: asyncio.Task | None = None
         self._decode_slots = asyncio.BoundedSemaphore(1)
 
@@ -81,8 +86,20 @@ class EnvClient:
         )
         try:
             data = await asyncio.wait_for(future, timeout)
-        except (TimeoutError, asyncio.CancelledError):
+        except TimeoutError:
+            self._pending.pop(request_id, None)
+            raise
+        except asyncio.CancelledError:
             self._pending.pop(request_id, None)
+            if request.method == RunRequest.method:
+                # Fire-and-forget: tell the server to abort the rollout so the
+                # episode stops consuming inference and env-runtime resources.
+                # A task on the loop outlives this (cancelled) caller.
+                task = asyncio.get_running_loop().create_task(
+                    self._send_cancel(request_id)
+                )
+                self._cancel_tasks.add(task)
+                task.add_done_callback(self._cancel_tasks.discard)
             raise
         if response_type is HealthResponse:
             response = response_type.model_validate(msgpack.unpackb(data, raw=False))
@@ -109,6 +126,17 @@ class EnvClient:
             raise RuntimeError(response.error or "env server request failed")
         return response
 
+    async def _send_cancel(self, run_request_id: str) -> None:
+        """Best-effort server-side abort of an abandoned run."""
+        with contextlib.suppress(Exception):
+            payload = msgpack.packb(
+                CancelRequest(request_id=run_request_id).model_dump(mode="json"),
+                use_bin_type=True,
+            )
+            await self.socket.send_multipart(
+                [uuid.uuid4().hex.encode(), CancelRequest.method.encode(), payload]
+            )
+
     async def health(self, timeout: float = 2.0) -> bool:
         try:
             return (
@@ -157,5 +185,10 @@ class EnvClient:
             with contextlib.suppress(asyncio.CancelledError):
                 await self._receiver
             self._receiver = None
+        # Let scheduled fire-and-forget cancels reach the socket first, or a
+        # run cancelled right before close() leaves its server-side rollout
+        # running. Loop: awaiting one wave can schedule another
+        while self._cancel_tasks:
+            await asyncio.gather(*list(self._cancel_tasks), return_exceptions=True)
         self.socket.close()
         self.ctx.term()

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/serve/pool.py

@@ -36,11 +36,14 @@ import zmq.asyncio
 
 from verifiers.v1.configs.env import EnvConfig
 from verifiers.v1.serve.server import EnvServer
-from verifiers.v1.serve.types import HealthResponse
+from verifiers.v1.serve.types import CancelResponse, HealthResponse
 
 logger = logging.getLogger(__name__)
 
 _HEALTH = msgpack.packb(HealthResponse().model_dump(mode="json"), use_bin_type=True)
+_CANCEL_MISS = msgpack.packb(
+    CancelResponse(cancelled=False).model_dump(mode="json"), use_bin_type=True
+)
 
 
 def _arm_teardown(death_pipe=None) -> None:
@@ -192,6 +195,32 @@ class EnvServerPool:
                         await self.frontend.send_multipart(
                             [client_id, request_id, _HEALTH]
                         )
+                    elif method == b"cancel":
+                        # Route the cancel to the worker holding the target run;
+                        # an unknown/finished target (or an unparseable payload
+                        # — one bad frame must not tear down the broker) is
+                        # answered inline
+                        try:
+                            body = msgpack.unpackb(payload, raw=False)
+                            target = str(body.get("request_id", "")).encode()
+                        except Exception:  # noqa: BLE001 - one bad frame must not kill the broker
+                            target = b""
+                        target_entry = pending.get(target)
+                        if target_entry is None:
+                            await self.frontend.send_multipart(
+                                [client_id, request_id, _CANCEL_MISS]
+                            )
+                        else:
+                            worker = target_entry["worker"]
+                            worker["active"] += 1
+                            pending[request_id] = {
+                                "client_id": client_id,
+                                "worker": worker,
+                            }
+                            in_flight += 1
+                            await worker["dealer"].send_multipart(
+                                [request_id, method, payload]
+                            )
                     else:
                         worker = min(self.workers, key=lambda w: w["active"])
                         worker["active"] += 1

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/serve/server.py

@@ -11,6 +11,8 @@ from verifiers.v1.configs.env import EnvConfig
 from verifiers.v1.serve.encoding import msgpack_encoder
 from verifiers.v1.serve.types import (
     BaseResponse,
+    CancelRequest,
+    CancelResponse,
     HealthResponse,
     RunRequest,
     RunResponse,
@@ -47,6 +49,8 @@ class EnvServer:
             )
         # This worker's episode bound (`--max-concurrent`), spanning requests.
         self._gate = asyncio.Semaphore(max_concurrent) if max_concurrent else None
+        # In-flight run tasks by wire request_id, so a `cancel` can abort them
+        self._running: dict[str, asyncio.Task] = {}
         self.ctx = zmq.asyncio.Context()
         self.frontend = self.ctx.socket(zmq.ROUTER)
         self.frontend.setsockopt(zmq.ROUTER_MANDATORY, 1)
@@ -107,11 +111,26 @@ class EnvServer:
             if route == "health":
                 response: BaseResponse = HealthResponse()
             elif route == "run":
-                response = await self._run(RunRequest.model_validate(raw))
+                # Registered in the dispatch loop, before this task first runs
+                try:
+                    response = await self._run(RunRequest.model_validate(raw))
+                finally:
+                    self._running.pop(request_id.decode(), None)
+            elif route == "cancel":
+                target = CancelRequest.model_validate(raw).request_id
+                running = self._running.get(target)
+                if running is not None:
+                    running.cancel()
+                    logger.info("cancelled run %s on client request", target)
+                response = CancelResponse(cancelled=running is not None)
             else:
                 response = BaseResponse(
                     success=False, error=f"unknown method {route!r}"
                 )
+        except asyncio.CancelledError:
+            # An aborted run still replies so broker/client accounting stays
+            # exact; the requesting client has already given up on the result
+            response = BaseResponse(success=False, error="Cancelled: rollout aborted")
         except (
             Exception
         ) as e:  # a failed request is data, not a crash — report and keep serving
@@ -156,6 +175,12 @@ class EnvServer:
                     task = asyncio.create_task(
                         self._handle(client_id, request_id, method, payload)
                     )
+                    if method == b"run":
+                        # Register at dispatch, not handler start: a cancel
+                        # processed ahead of the run's handler task must still
+                        # find its target (ZMQ preserves per-client frame
+                        # order, so the run always arrives first)
+                        self._running[request_id.decode()] = task
                     tasks.add(task)
                     task.add_done_callback(tasks.discard)
             except (asyncio.CancelledError, KeyboardInterrupt):

{verifiers-0.3.1.dev53 → verifiers-0.3.1.dev55}/verifiers/v1/serve/types.py

@@ -26,6 +26,19 @@ class HealthResponse(BaseResponse):
     pass
 
 
+class CancelRequest(BaseRequest):
+    """Abort an in-flight run by its wire ``request_id``. Idempotent — a
+    finished or unknown run cancels successfully with ``cancelled=False``."""
+
+    method: ClassVar[str] = "cancel"
+    request_id: str
+
+
+class CancelResponse(BaseResponse):
+    cancelled: bool = False
+    """Whether a live run was found and aborted."""
+
+
 class RunRequest(BaseRequest):
     """One env-rollout, shipping the task itself: `task_data` is the dumped
     `TaskData` the server validates into the taskset's declared type."""