verifiers 0.1.15.dev367__tar.gz → 0.1.15.dev369__tar.gz

{verifiers-0.1.15.dev367 → verifiers-0.1.15.dev369}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev367
+Version: 0.1.15.dev369
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev367 → verifiers-0.1.15.dev369}/verifiers/v1/cli/eval/runner.py

@@ -182,8 +182,13 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
                 config.model,
             )
         logger.info("results: %s", out)
+        request_concurrency = config.max_concurrent
+        if request_concurrency and info.requires_group_scoring:
+            # max_concurrent is a rollout resource bound, not a request-throughput target.
+            # A group is indivisible, so one oversized group must still be allowed to run.
+            request_concurrency = max(1, request_concurrency // config.num_rollouts)
         semaphore = (
-            asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
+            asyncio.Semaphore(request_concurrency) if request_concurrency else None
         )
         write_lock = asyncio.Lock()
 

{verifiers-0.1.15.dev367 → verifiers-0.1.15.dev369}/verifiers/v1/mcp/multiplex.py

@@ -31,6 +31,7 @@ to absolute paths outside the private CWD are not isolated.
 from __future__ import annotations
 
 import asyncio
+import concurrent.futures
 import contextlib
 import hashlib
 import logging
@@ -148,8 +149,7 @@ def serve_forked(app, sock: socket.socket, server) -> None:
         child = children.get(key)
         if child is not None and child.ready.is_set():
             return child
-        # The lock is held ONLY for the synchronous fork()+register, never the readiness wait — so a
-        # cold fork serializes other forks (fork-safety) but doesn't stall traffic to other children.
+        # Readiness waits stay outside the lock, so traffic to ready children remains lock-free.
         async with forking:
             child = children.get(key)
             creating = child is None
@@ -177,15 +177,29 @@ def serve_forked(app, sock: socket.socket, server) -> None:
         return child
 
     async def reap(key: str) -> None:
-        child = children.pop(key, None)
-        if not child:
-            return
-        child.ready.set()  # wake any waiter blocked in `ensure` — it re-checks `children` and re-forks
-        with contextlib.suppress(Exception):
-            os.kill(child.pid, signal.SIGKILL)
-        with contextlib.suppress(Exception):
-            os.waitpid(child.pid, 0)
-        shutil.rmtree(child.cwd, ignore_errors=True)
+        # Keep fork() out of the cleanup thread's lifetime; ready children bypass this lock.
+        async with forking:
+            child = children.pop(key, None)
+            if not child:
+                return
+            child.ready.set()  # wake waiters; they re-check `children` before re-forking
+            with contextlib.suppress(Exception):
+                os.kill(child.pid, signal.SIGKILL)
+
+            def cleanup() -> None:
+                with contextlib.suppress(Exception):
+                    os.waitpid(child.pid, 0)
+                shutil.rmtree(child.cwd, ignore_errors=True)
+
+            # One worker keeps cleanup ordered and lets it finish if this task is cancelled.
+            loop = asyncio.get_running_loop()
+            with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+                future = loop.run_in_executor(executor, cleanup)
+                try:
+                    await asyncio.shield(future)
+                except asyncio.CancelledError:
+                    await future
+                    raise
         logger.info("fork: reaped child pid=%d", child.pid)
 
     async def _respond(send, status: int, body: bytes) -> None:

{verifiers-0.1.15.dev367 → verifiers-0.1.15.dev369}/verifiers/v1/serve/pool.py

@@ -37,7 +37,7 @@ import zmq.asyncio
 
 from verifiers.v1.env import EnvConfig
 from verifiers.v1.serve.server import EnvServer
-from verifiers.v1.serve.types import HealthResponse
+from verifiers.v1.serve.types import HealthResponse, RunGroupRequest
 
 logger = logging.getLogger(__name__)
 
@@ -161,7 +161,7 @@ class EnvServerPool:
             self._poller.register(dealer, zmq.POLLIN)
 
     def _maybe_scale_up(self, in_flight: int) -> None:
-        """Spawn one more worker when in-flight requests reach 90% of current capacity.
+        """Spawn one more worker when in-flight rollout slots reach 90% of capacity.
 
         A new worker starts at `active=0`, so least-busy dispatch funnels the backlog to
         it as it comes online (a few seconds to load the env) — fine, since we only scale
@@ -188,7 +188,8 @@ class EnvServerPool:
         # (`max_workers` is a concrete count when elastic is off).
         for _ in range(1 if self.elastic else (self.max_workers or 1)):
             self._spawn_worker()
-        pending: dict[bytes, dict] = {}  # request_id -> {client_id, worker}
+        # request_id -> {client_id, worker, rollout_slots}
+        pending: dict[bytes, dict] = {}
         logger.info(
             "EnvServerPool up: address=%s workers=%d/%s multiplex=%d elastic=%s",
             self.address,
@@ -198,6 +199,7 @@ class EnvServerPool:
             self.elastic,
         )
         try:
+            in_flight = 0
             while True:
                 events = dict(await self._poller.poll())
                 if self.frontend in events:
@@ -212,16 +214,29 @@ class EnvServerPool:
                             [client_id, request_id, _HEALTH]
                         )
                     else:
+                        # Pool capacity is measured in rollouts; one group request carries n.
+                        rollout_slots = 1
+                        if method == b"run_group":
+                            with contextlib.suppress(Exception):
+                                request = RunGroupRequest.model_validate(
+                                    msgpack.unpackb(payload, raw=False)
+                                )
+                                rollout_slots = max(1, request.n)
                         worker = min(self.workers, key=lambda w: w["active"])
-                        worker["active"] += 1
-                        pending[request_id] = {"client_id": client_id, "worker": worker}
+                        worker["active"] += rollout_slots
+                        pending[request_id] = {
+                            "client_id": client_id,
+                            "worker": worker,
+                            "rollout_slots": rollout_slots,
+                        }
+                        in_flight += rollout_slots
                         # forward without client_id — the DEALER identity is the worker's
                         # `client_id`; we hold the real one in `pending`.
                         await worker["dealer"].send_multipart(
                             [request_id, method, payload]
                         )
                         if self.elastic:
-                            self._maybe_scale_up(len(pending))
+                            self._maybe_scale_up(in_flight)
                 for w in self.workers:
                     if w["dealer"] in events:
                         request_id, data = await w["dealer"].recv_multipart(copy=False)
@@ -229,7 +244,8 @@ class EnvServerPool:
                         entry = pending.pop(request_id.bytes, None)
                         if entry is None:
                             continue
-                        entry["worker"]["active"] -= 1
+                        entry["worker"]["active"] -= entry["rollout_slots"]
+                        in_flight -= entry["rollout_slots"]
                         with contextlib.suppress(zmq.ZMQError):
                             await self.frontend.send_multipart(
                                 [entry["client_id"], request_id, data], copy=False