verifiers 0.1.15.dev157__tar.gz → 0.1.15.dev158__tar.gz

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev158}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev157
+Version: 0.1.15.dev158
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev158}/tests/test_v1_runtime_lifecycle.py

@@ -18,7 +18,7 @@ from pydantic import BaseModel
 
 import verifiers as vf
 from verifiers.clients import Client
-from verifiers.types import ClientConfig
+from verifiers.types import ClientConfig, Messages
 from verifiers.types import Response, ResponseMessage, ToolCall
 from verifiers.types import Tool
 from verifiers.types import Usage
@@ -98,6 +98,18 @@ class CapturingModelClient(FakeModelClient):
         return await super().get_response(**kwargs)
 
 
+class BlockingModelClient(CapturingModelClient):
+    def __init__(self, responses: list[Response]):
+        super().__init__(responses)
+        self.entered = asyncio.Event()
+        self.release = asyncio.Event()
+
+    async def get_response(self, **kwargs: object) -> Response:
+        self.entered.set()
+        await self.release.wait()
+        return await super().get_response(**kwargs)
+
+
 class FakeCreateSandboxRequest:
     def __init__(self, **kwargs: object):
         self.kwargs = kwargs
@@ -537,6 +549,37 @@ async def endpoint_trajectory_program(task, state):
     return state
 
 
+async def concurrent_endpoint_program(task, state):
+    _ = task
+    root = state["endpoint_root_url"].rstrip("/")
+    config = state.get_endpoint_config(api="chat")
+    endpoint_client = cast(OpenAI, state.get_client(api="chat", sync=True))
+    api_key = endpoint_client.api_key
+    endpoint_client.close()
+
+    def post_chat(content: str) -> dict[str, object]:
+        payload = {
+            "model": config.model,
+            "messages": [{"role": "user", "content": content}],
+        }
+        request = urllib.request.Request(
+            f"{root}/v1/chat/completions",
+            data=json.dumps(payload).encode(),
+            headers={
+                "content-type": "application/json",
+                "Authorization": f"Bearer {api_key}",
+            },
+        )
+        with urllib.request.urlopen(request) as response:
+            return json.loads(response.read().decode())
+
+    state["endpoint_concurrent_responses"] = await asyncio.gather(
+        asyncio.to_thread(post_chat, "first"),
+        asyncio.to_thread(post_chat, "second"),
+    )
+    return state
+
+
 async def mcp_proxy_program(task, state):
     _ = task
     from mcp import ClientSession, StdioServerParameters
@@ -683,6 +726,7 @@ for _name, _value in {
     "child_reads_program_sandbox": child_reads_program_sandbox,
     "endpoint_program": endpoint_program,
     "endpoint_trajectory_program": endpoint_trajectory_program,
+    "concurrent_endpoint_program": concurrent_endpoint_program,
     "mcp_proxy_program": mcp_proxy_program,
     "child_program": child_program,
     "parent_program": parent_program,
@@ -808,6 +852,36 @@ async def test_endpoint_request_can_hide_internal_model_call_from_trajectory() -
     )
 
 
+@pytest.mark.asyncio
+async def test_endpoint_max_turns_counts_inflight_visible_requests() -> None:
+    client = BlockingModelClient(
+        [fake_response("allowed"), fake_response("unexpected")]
+    )
+    harness = make_harness(
+        program={"fn": program_ref("concurrent_endpoint_program")},
+        client=client,
+        model="test-model",
+        max_turns=1,
+    )
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    run_task = asyncio.create_task(harness.run(task))
+    await asyncio.wait_for(client.entered.wait(), timeout=1.0)
+    await asyncio.sleep(0.05)
+    client.release.set()
+    state = await run_task
+    await harness.teardown()
+
+    contents = [
+        response["choices"][0]["message"]["content"]
+        for response in state["endpoint_concurrent_responses"]
+    ]
+    assert sorted(contents) == ["", "allowed"]
+    assert len(client.requests) == 1
+    assert len(state["trajectory"]) == 1
+    assert state["stop_condition"] == "max_turns_reached"
+
+
 @pytest.mark.asyncio
 async def test_state_helpers_load_runtime_tools_while_rollout_is_active() -> None:
     harness = make_harness(
@@ -950,6 +1024,38 @@ async def test_base_program_submits_system_prompt_before_prompt() -> None:
     assert state["completion"][-1]["content"] == "ok"
 
 
+@pytest.mark.asyncio
+async def test_base_program_max_turns_uses_stop_condition() -> None:
+    client = CapturingModelClient(
+        [fake_response(content="done"), fake_response(content="unexpected")]
+    )
+    harness = make_harness(client=cast(Client, client), model="fake", max_turns=1)
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    state = await harness.run(task)
+
+    assert len(client.requests) == 1
+    assert len(state["trajectory"]) == 1
+    assert state["stop_condition"] == "max_turns_reached"
+
+
+@pytest.mark.asyncio
+async def test_model_request_reservation_released_when_client_resolution_fails() -> (
+    None
+):
+    harness = make_harness(model="fake", max_turns=1)
+    task = vf.Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+    state = await harness.setup_state(task, vf.State.for_task(task))
+
+    with pytest.raises(RuntimeError, match="no model client"):
+        await harness.runtime.submit_model_request(
+            cast(Messages, state["prompt"]), task, state
+        )
+
+    assert harness.runtime.visible_model_requests(state) == 0
+    assert state["trajectory"] == []
+
+
 @pytest.mark.asyncio
 async def test_taskset_setup_initializes_base_harness_prompt_and_sampling() -> None:
     class SetupRuntimeTaskset(vf.Taskset):

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev158}/verifiers/v1/harness.py

@@ -314,6 +314,11 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
             raise TypeError("state.trajectory must be a list.")
         return float(len(trajectory))
 
+    @vf.stop
+    async def max_turns_reached(self, state: State) -> bool:
+        max_turns = state.get_max_turns(self.config.max_turns)
+        return max_turns > 0 and self.runtime.visible_model_requests(state) >= max_turns
+
     async def setup_state(self, task: Task, state: State) -> State:
         await self.setup_runtime_state(task, state)
         await self.setup_model_state(task, state)
@@ -547,9 +552,6 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
             sync_completion()
             if await self.runtime.is_completed(task, state):
                 return state
-            if max_turns > 0 and turn >= max_turns:
-                state._set_stop_condition("max_turns_reached", overwrite=True)
-                return state
         return state
 
     def command_program(

{verifiers-0.1.15.dev157 → verifiers-0.1.15.dev158}/verifiers/v1/runtime.py

@@ -21,7 +21,7 @@ from typing import (
 )
 
 from verifiers.clients import Client, resolve_client
-from verifiers.types import Messages, Response, Tool
+from verifiers.types import Messages, Response, ResponseMessage, Tool
 from verifiers.types import ClientConfig, ClientType, SamplingArgs
 from verifiers.utils.async_utils import maybe_call_with_named_args
 from verifiers.utils.client_utils import resolve_client_config
@@ -226,40 +226,7 @@ class Runtime:
             if owner is not None:
                 self.toolsets.extend(iter_toolsets(owner.toolsets))
         self.named_toolsets = self._collect_named_toolsets()
-        self.scoped_tools: dict[tuple[int, str, str], list[ToolEntry]] = {}
-        self.runtime_objects: dict[RuntimeObjectOwner, RuntimeObjectStore] = {
-            "toolset": {},
-            "user": {},
-            "taskset": {},
-            "harness": {},
-        }
-        self.model_clients: dict[str, Client] = {}
-        self.owned_model_clients: set[str] = set()
-        self._sandbox_client = None
-        self.sandbox_leases: dict[tuple[str, str], SandboxLease] = {}
-        self.sandbox_creation_tasks: dict[
-            tuple[str, str], asyncio.Task[SandboxLease]
-        ] = {}
-        self.upload_archive_tasks: dict[tuple[str, str, str], asyncio.Task[Path]] = {}
-        self.sandbox_lock = asyncio.Lock()
-        sandbox_config = (
-            self.harness.sandbox
-            if self.harness is not None and self.harness.sandbox is not None
-            else SandboxConfig()
-        )
-        create_concurrency = sandbox_config.create_concurrency
-        create_rate = sandbox_config.create_rate_per_second
-        delete_concurrency = sandbox_config.delete_concurrency
-        delete_rate = sandbox_config.delete_rate_per_second
-        self.sandbox_create_semaphore = asyncio.Semaphore(create_concurrency)
-        self.sandbox_create_rate_limiter = AsyncRateLimiter(create_rate)
-        self.sandbox_delete_semaphore = asyncio.Semaphore(delete_concurrency)
-        self.sandbox_delete_rate_limiter = AsyncRateLimiter(delete_rate)
-        self.mcp_exit_stacks: dict[str, AsyncExitStack] = {}
-        self.mcp_tools: dict[str, RuntimeTools] = {}
-        self.mcp_tool_parents: dict[str, dict[str, tuple[Toolset, ...]]] = {}
-        self.trajectories: dict[str, list[JsonData]] = {}
-        self.tool_handles: dict[str, tuple[Task, State, tuple[str, ...]]] = {}
+
         self.stop_conditions = collect_handlers(
             self._handler_owners(),
             "stop",
@@ -326,6 +293,46 @@ class Runtime:
             ),
         )
 
+        self.trajectories: dict[str, list[JsonData]] = {}
+        self.model_clients: dict[str, Client] = {}
+        self.owned_model_clients: set[str] = set()
+        self._model_request_locks: dict[str, asyncio.Lock] = {}
+        self._inflight_visible_model_requests: dict[str, int] = {}
+
+        self.scoped_tools: dict[tuple[int, str, str], list[ToolEntry]] = {}
+        self.tool_handles: dict[str, tuple[Task, State, tuple[str, ...]]] = {}
+        self.runtime_objects: dict[RuntimeObjectOwner, RuntimeObjectStore] = {
+            "toolset": {},
+            "user": {},
+            "taskset": {},
+            "harness": {},
+        }
+
+        self._sandbox_client = None
+        self.sandbox_leases: dict[tuple[str, str], SandboxLease] = {}
+        self.sandbox_creation_tasks: dict[
+            tuple[str, str], asyncio.Task[SandboxLease]
+        ] = {}
+        self.upload_archive_tasks: dict[tuple[str, str, str], asyncio.Task[Path]] = {}
+        self.sandbox_lock = asyncio.Lock()
+        sandbox_config = (
+            self.harness.sandbox
+            if self.harness is not None and self.harness.sandbox is not None
+            else SandboxConfig()
+        )
+        create_concurrency = sandbox_config.create_concurrency
+        create_rate = sandbox_config.create_rate_per_second
+        delete_concurrency = sandbox_config.delete_concurrency
+        delete_rate = sandbox_config.delete_rate_per_second
+        self.sandbox_create_semaphore = asyncio.Semaphore(create_concurrency)
+        self.sandbox_create_rate_limiter = AsyncRateLimiter(create_rate)
+        self.sandbox_delete_semaphore = asyncio.Semaphore(delete_concurrency)
+        self.sandbox_delete_rate_limiter = AsyncRateLimiter(delete_rate)
+
+        self.mcp_exit_stacks: dict[str, AsyncExitStack] = {}
+        self.mcp_tools: dict[str, RuntimeTools] = {}
+        self.mcp_tool_parents: dict[str, dict[str, tuple[Toolset, ...]]] = {}
+
     @property
     def has_group_signals(self) -> bool:
         return bool(self.group_signals)
@@ -438,6 +445,19 @@ class Runtime:
             list[JsonData], trajectory
         )
 
+    def visible_model_requests(self, state: State) -> int:
+        trajectory = state.get("trajectory") or []
+        if not isinstance(trajectory, list):
+            raise TypeError("state.trajectory must be a list.")
+        trajectory_id = str(state["trajectory_id"])
+        completed = sum(
+            1
+            for step in trajectory
+            if isinstance(step, dict)
+            and str(step.get("trajectory_id")) == trajectory_id
+        )
+        return completed + self._inflight_visible_model_requests.get(trajectory_id, 0)
+
     def resolved_handles(self, state: State) -> ResolvedRuntimeHandlesConfig:
         runtime = state.runtime_state()
         resolved = runtime.get("resolved") or {}
@@ -858,41 +878,92 @@ class Runtime:
         context: ModelRequestContext | None = None,
     ) -> Response:
         context = context or ModelRequestContext()
-        client = self.model_client(state)
-        request_start = time.time()
-        response = await client.get_response(
-            prompt=prompt,
+        reserved = await self._reserve_model_request(task, state, context)
+        if not reserved:
+            return self._completed_model_response(state)
+        released = False
+        try:
+            client = self.model_client(state)
+            request_start = time.time()
+            response = await client.get_response(
+                prompt=prompt,
+                model=self.model(state),
+                tools=tool_defs,
+                sampling_args=self.sampling_args(state),
+                state=state,
+            )
+            request_end = time.time()
+            state.record_model_timing(request_start, request_end)
+            record_response_usage(state, response)
+            completion = await parse_response_message(response)
+            tokens = await parse_response_tokens(response)
+            is_truncated = response.message.is_truncated or (
+                tokens is not None and bool(tokens.get("is_truncated"))
+            )
+            step = {
+                "prompt": serializable(prompt),
+                "completion": serializable(completion),
+                "response": serializable(response),
+                "tokens": serializable(tokens),
+                "reward": None,
+                "advantage": None,
+                "is_truncated": bool(is_truncated),
+                "trajectory_id": str(state["trajectory_id"]),
+                "extras": context.extras(),
+            }
+            if context.trajectory_visibility == "append":
+                state["trajectory"].append(step)
+                self._release_model_request(state, context)
+                released = True
+                await self.is_completed(task, state)
+            elif context.trajectory_visibility != "hidden":
+                raise AssertionError(
+                    f"Unknown trajectory visibility: {context.trajectory_visibility!r}"
+                )
+            return response
+        finally:
+            if not released:
+                self._release_model_request(state, context)
+
+    async def _reserve_model_request(
+        self, task: Task, state: State, context: ModelRequestContext
+    ) -> bool:
+        key = str(state["trajectory_id"])
+        lock = self._model_request_locks.setdefault(key, asyncio.Lock())
+        async with lock:
+            if await self.is_completed(task, state):
+                return False
+            if context.trajectory_visibility != "append":
+                return True
+            self._inflight_visible_model_requests[key] = (
+                self._inflight_visible_model_requests.get(key, 0) + 1
+            )
+            return True
+
+    def _release_model_request(
+        self, state: State, context: ModelRequestContext
+    ) -> None:
+        if context.trajectory_visibility != "append":
+            return
+        key = str(state["trajectory_id"])
+        count = self._inflight_visible_model_requests.get(key, 0)
+        if count <= 1:
+            self._inflight_visible_model_requests.pop(key, None)
+        else:
+            self._inflight_visible_model_requests[key] = count - 1
+
+    def _completed_model_response(self, state: State) -> Response:
+        return Response(
+            id=f"completed_{uuid.uuid4().hex}",
+            created=int(time.time()),
             model=self.model(state),
-            tools=tool_defs,
-            sampling_args=self.sampling_args(state),
-            state=state,
-        )
-        request_end = time.time()
-        state.record_model_timing(request_start, request_end)
-        record_response_usage(state, response)
-        completion = await parse_response_message(response)
-        tokens = await parse_response_tokens(response)
-        is_truncated = response.message.is_truncated or (
-            tokens is not None and bool(tokens.get("is_truncated"))
+            message=ResponseMessage(
+                role="assistant",
+                content="",
+                finish_reason="stop",
+                is_truncated=False,
+            ),
         )
-        step = {
-            "prompt": serializable(prompt),
-            "completion": serializable(completion),
-            "response": serializable(response),
-            "tokens": serializable(tokens),
-            "reward": None,
-            "advantage": None,
-            "is_truncated": bool(is_truncated),
-            "trajectory_id": str(state["trajectory_id"]),
-            "extras": context.extras(),
-        }
-        if context.trajectory_visibility == "append":
-            state["trajectory"].append(step)
-        elif context.trajectory_visibility != "hidden":
-            raise AssertionError(
-                f"Unknown trajectory visibility: {context.trajectory_visibility!r}"
-            )
-        return response
 
     async def setup_rollout(
         self,
@@ -978,6 +1049,9 @@ class Runtime:
         await self.close_mcp_tools(state)
         self.release_scoped_tools("rollout", state)
         await self.release_model_client(state)
+        key = str(state["trajectory_id"])
+        self._model_request_locks.pop(key, None)
+        self._inflight_visible_model_requests.pop(key, None)
         self.release_tool_handles(state)
 
     async def cleanup_group(self, tasks: list[Task], states: list[State]) -> None: