verifiers 0.4.1.dev44__tar.gz → 0.4.1.dev45__tar.gz

{verifiers-0.4.1.dev44 → verifiers-0.4.1.dev45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev44
+Version: 0.4.1.dev45
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.4.1.dev45/tests/integration/fixtures/parallel_v1.py

@@ -0,0 +1,84 @@
+"""parallel-v1: one rollout whose harness sends `n` chat programs at the model at once,
+so their requests overlap on one trace. With `same_prompt`, every copy sends the task
+prompt; otherwise copy `i` appends its index, so the copies diverge after the system
+prompt."""
+
+import asyncio
+
+from pydantic import PositiveInt
+
+import verifiers.v1 as vf
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.harnesses.utils.launch import (
+    CHAT_PROGRAM_SOURCE,
+    launch_chat_program,
+)
+from verifiers.v1.runtimes import ProgramResult, Runtime
+
+__all__ = ["ParallelHarness", "ParallelTaskset"]
+
+
+class ParallelHarnessConfig(HarnessConfig):
+    n: PositiveInt = 4
+    same_prompt: bool = True
+
+
+class ParallelHarness(Harness[ParallelHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    EXECUTES_CODE = False
+    NEEDS_CONTAINER = False
+
+    async def setup(self, runtime: Runtime) -> None:
+        await runtime.prepare_uv_script(CHAT_PROGRAM_SOURCE, self.config.resolved_env)
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: vf.Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: vf.TaskData,
+        tool_interception_url: str | None = None,
+    ) -> ProgramResult:
+        system_prompt, prompt = self.resolve_prompt(data)
+
+        def copy_prompt(i: int) -> str:
+            return prompt if self.config.same_prompt else f"{prompt} ({i})"
+
+        results = await asyncio.gather(
+            *(
+                launch_chat_program(
+                    CHAT_PROGRAM_SOURCE,
+                    self.config,
+                    ctx,
+                    trace,
+                    runtime,
+                    endpoint,
+                    secret,
+                    mcp_urls,
+                    system_prompt,
+                    copy_prompt(i),
+                )
+                for i in range(self.config.n)
+            )
+        )
+        return next((r for r in results if r.exit_code != 0), results[0])
+
+
+class ParallelTask(vf.Task):
+    @vf.reward(weight=1.0)
+    async def answered(self, trace: vf.Trace) -> float:
+        return float(trace.num_turns >= 1)
+
+
+class ParallelTaskset(vf.Taskset[ParallelTask, vf.TasksetConfig]):
+    def load(self) -> list[ParallelTask]:
+        return [
+            ParallelTask(
+                vf.TaskData(idx=0, system_prompt="Be brief.", prompt="Reply with OK.")
+            )
+        ]

verifiers-0.4.1.dev45/tests/integration/test_parallel.py

@@ -0,0 +1,36 @@
+"""Overlapping requests on one trace: a harness sends `n` chat programs at a scripted
+upstream at once, and the trace graph keeps one branch per copy that shares exactly the
+prefix the copies have in common."""
+
+import pytest
+from parallel_v1 import ParallelHarnessConfig
+
+N = 4
+
+
+@pytest.mark.parametrize("harness_runtime", ["subprocess"], indirect=True)
+@pytest.mark.parametrize("same_prompt", [True, False], ids=["same", "distinct"])
+async def test_parallel_requests(run, harness_runtime, scripted_model, same_prompt):
+    (episode,) = await run(
+        "parallel-v1",
+        harness=ParallelHarnessConfig(id="parallel-v1", n=N, same_prompt=same_prompt),
+        runtime=harness_runtime,
+        client=scripted_model,
+    )
+    (trace,) = episode.traces
+    assert trace.ok, [error.message for error in trace.errors]
+    assert trace.num_turns == trace.num_branches == len(trace.calls) == N
+    assert len({call.node for call in trace.calls}) == N
+
+    roles = [node.message.role for node in trace.nodes]
+    assert roles.count("system") == 1
+    users = [i for i, role in enumerate(roles) if role == "user"]
+    replies = [node for node in trace.nodes if node.message.role == "assistant"]
+    # The same prompt shares the system and user nodes; distinct prompts share only
+    # the system node.
+    assert len(users) == (1 if same_prompt else N)
+    assert len(trace.nodes) == 1 + len(users) + N
+    assert {trace.nodes[i].parent for i in users} == {0}
+    assert sorted(reply.parent for reply in replies) == sorted(
+        users * N if same_prompt else users
+    )