verifiers 0.1.12.dev0__tar.gz → 0.1.12.dev2__tar.gz

{verifiers-0.1.12.dev0 → verifiers-0.1.12.dev2}/.gitignore

@@ -11,6 +11,7 @@ uv.lock
 .scratch/
 .chroma_db/
 /.codex/environments/
+.python-version
 
 # artifacts
 core.* 

{verifiers-0.1.12.dev0 → verifiers-0.1.12.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.12.dev0
+Version: 0.1.12.dev2
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -34,12 +34,13 @@ Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: numpy
 Requires-Dist: openai-agents>=0.0.7
 Requires-Dist: openai>=1.108.1
-Requires-Dist: prime-sandboxes>=0.2.16
-Requires-Dist: prime-tunnel>=0.1.4
+Requires-Dist: prime-sandboxes>=0.2.19
+Requires-Dist: prime-tunnel>=0.1.5
 Requires-Dist: pydantic>=2.11.9
 Requires-Dist: pyzmq>=27.1.0
 Requires-Dist: requests
 Requires-Dist: rich
+Requires-Dist: setproctitle>=1.3.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: textual
 Requires-Dist: tomli; python_version < '3.11'

{verifiers-0.1.12.dev0 → verifiers-0.1.12.dev2}/pyproject.toml

@@ -37,8 +37,8 @@ dependencies = [
     "nest-asyncio>=1.6.0", # for jupyter notebooks
     "openai>=1.108.1",
     "openai-agents>=0.0.7",
-    "prime-tunnel>=0.1.4",
-    "prime-sandboxes>=0.2.16",
+    "prime-tunnel>=0.1.5",
+    "prime-sandboxes>=0.2.19",
     "pydantic>=2.11.9",
     "requests",
     "rich",
@@ -51,6 +51,7 @@ dependencies = [
     "pyzmq>=27.1.0",
     "msgpack>=1.1.2",
     "aiolimiter>=1.2.1",
+    "setproctitle>=1.3.0",
 ]
 
 [dependency-groups]
@@ -104,6 +105,7 @@ rl = [
 
 [tool.uv]
 preview = true
+required-version = "<0.11.0"
 
 [tool.uv.extra-build-dependencies]
 flash-attn = [{ requirement = "torch", match-runtime = true }]

verifiers-0.1.12.dev2/tests/test_composable_env.py

@@ -0,0 +1,200 @@
+"""Tests for the composable architecture: Task, TaskSet, SandboxTaskSet, SandboxSpec."""
+
+import pytest
+
+import verifiers as vf
+from verifiers.envs.experimental.composable import (
+    ComposableEnv,
+    Harness,
+    SandboxSpec,
+    SandboxTaskSet,
+    Task,
+    TaskSet,
+)
+
+
+# ── Mock Rubrics ──────────────────────────────────────────────────────
+
+
+class MockSandboxRubric(vf.Rubric):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_reward_func(self.solved)
+
+    async def solved(self, state, **kwargs) -> float:
+        return 1.0 if state.get("test_output") == "PASS" else 0.0
+
+
+class MockMathRubric(vf.Rubric):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_reward_func(self.correct)
+
+    async def correct(self, state, **kwargs) -> float:
+        return 1.0 if state.get("info", {}).get("id") == 0 else 0.0
+
+
+# ── Mock TaskSets ───────────────────────────────────────────────────────
+
+
+class MockSandboxTaskSet(SandboxTaskSet):
+    """SandboxTaskSet for testing."""
+
+    def get_instruction(self, info):
+        return f"Fix bug #{info.get('id', 0)}"
+
+    def get_sandbox_spec(self, info):
+        return SandboxSpec(image="python:3.11-slim", cpu_cores=2, memory_gb=2)
+
+    def get_rubric(self):
+        return MockSandboxRubric()
+
+    def get_workdir(self, info):
+        return "/testbed"
+
+    def get_env_vars(self):
+        return {"FOO": "bar"}
+
+
+class MockTaskSet(TaskSet):
+    """Plain TaskSet (no sandbox) for testing."""
+
+    def get_instruction(self, info):
+        return info.get("question", "")
+
+    def get_rubric(self):
+        return MockMathRubric()
+
+
+def _make_dataset(n=3):
+    from datasets import Dataset
+
+    return Dataset.from_dict(
+        {
+            "info": [{"id": i, "question": f"q{i}"} for i in range(n)],
+            "answer": ["" for _ in range(n)],
+        }
+    )
+
+
+# ── SandboxSpec ─────────────────────────────────────────────────────────
+
+
+def test_sandbox_spec_defaults():
+    spec = SandboxSpec()
+    assert spec.image == "python:3.11-slim"
+    assert spec.cpu_cores == 4
+
+
+def test_sandbox_spec_custom():
+    spec = SandboxSpec(image="lean-tactic:v4.27", gpu_count=1)
+    assert spec.image == "lean-tactic:v4.27"
+    assert spec.gpu_count == 1
+
+
+# ── Task from SandboxTaskSet ───────────────────────────────────────────
+
+
+def test_task_sandbox_spec():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert isinstance(task, Task)
+    assert task.sandbox_spec is not None
+    assert task.sandbox_spec.image == "python:3.11-slim"
+    assert task.sandbox_spec.cpu_cores == 2
+
+
+def test_task_image():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert task.image == "python:3.11-slim"
+
+
+def test_task_workdir():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert task.workdir == "/testbed"
+
+
+def test_task_repr_sandbox():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    task = ts[0]
+    assert "python:3.11-slim" in repr(task)
+
+
+# ── Task from plain TaskSet ────────────────────────────────────────────
+
+
+def test_task_no_sandbox():
+    ts = MockTaskSet(dataset=_make_dataset(), name="math")
+    task = ts[0]
+    assert task.sandbox_spec is None
+    assert task.image is None
+
+
+def test_task_repr_no_sandbox():
+    ts = MockTaskSet(dataset=_make_dataset(), name="math")
+    task = ts[0]
+    assert "no sandbox" in repr(task)
+
+
+# ── TaskSet ─────────────────────────────────────────────────────────────
+
+
+def test_taskset_isinstance():
+    ts = MockTaskSet(dataset=_make_dataset(), name="math")
+    assert not isinstance(ts, SandboxTaskSet)
+
+    ts2 = MockSandboxTaskSet(dataset=_make_dataset(), name="swe")
+    assert isinstance(ts2, SandboxTaskSet)
+
+
+def test_taskset_len():
+    ts = MockTaskSet(dataset=_make_dataset(5), name="test")
+    assert len(ts) == 5
+
+
+def test_taskset_iter():
+    ts = MockTaskSet(dataset=_make_dataset(3), name="test")
+    tasks = list(ts)
+    assert len(tasks) == 3
+    assert all(isinstance(t, Task) for t in tasks)
+
+
+def test_taskset_filter():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(5), name="test")
+    filtered = ts.filter(lambda ex: ex["info"]["id"] < 3)
+    assert len(filtered) == 3
+    assert isinstance(filtered, MockSandboxTaskSet)
+
+
+def test_taskset_take():
+    ts = MockSandboxTaskSet(dataset=_make_dataset(5), name="test")
+    taken = ts.take(2)
+    assert len(taken) == 2
+    assert isinstance(taken, MockSandboxTaskSet)
+
+
+def test_taskset_repr():
+    ts = MockTaskSet(dataset=_make_dataset(), name="mytest")
+    assert "mytest" in repr(ts)
+    assert "3" in repr(ts)
+
+
+@pytest.mark.asyncio
+async def test_composable_env_exports_task_workdir():
+    taskset = MockSandboxTaskSet(dataset=_make_dataset(), name="test")
+    env = ComposableEnv(
+        taskset=taskset,
+        harness=Harness(run_command="true"),
+    )
+
+    env_vars = await env.build_env_vars(
+        {
+            "info": {"id": 0},
+            "interception_base_url": "https://test.trycloudflare.com/v1",
+        }
+    )
+
+    assert env_vars["AGENT_WORKDIR"] == "/testbed"
+    assert env_vars["FOO"] == "bar"

{verifiers-0.1.12.dev0 → verifiers-0.1.12.dev2}/tests/test_env_server.py

@@ -4,7 +4,7 @@ Covers:
 - Health-check state transitions (STARTUP -> HEALTHY -> UNHEALTHY)
 - Request retry on ServerError and recovery timeouts
 - Server startup waiting
-- Cancellation propagation (client -> server)
+- Cancellation propagation (client -> router -> worker)
 """
 
 import asyncio
@@ -15,16 +15,16 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 
 from verifiers.types import ClientConfig, RolloutInput, UserMessage
-from verifiers.utils.worker_utils import get_free_port_pair
-from verifiers.workers.client.zmq_env_client import ZMQEnvClient
-from verifiers.workers.server.zmq_env_server import ZMQEnvServer
-from verifiers.workers.types import (
+from verifiers.utils.serve_utils import get_free_port
+from verifiers.serve import (
     HealthRequest,
     HealthResponse,
     PendingRequest,
     RunRolloutRequest,
     RunRolloutResponse,
     ServerState,
+    ZMQEnvClient,
+    ZMQEnvServer,
 )
 
 
@@ -36,7 +36,7 @@ def make_client(address: str = "tcp://127.0.0.1:5555", **kwargs) -> ZMQEnvClient
 
 def make_mock_server(address: str) -> ZMQEnvServer:
     """Create a ZMQEnvServer with a mocked environment (no real env loading)."""
-    with patch("verifiers.workers.server.env_server.vf") as mock_vf:
+    with patch("verifiers.serve.server.env_server.vf") as mock_vf:
         mock_env = MagicMock()
         mock_env._teardown = AsyncMock()
         mock_vf.load_environment.return_value = mock_env
@@ -77,20 +77,22 @@ def make_pending_request(
 
 
 @contextlib.asynccontextmanager
-async def run_server_and_client(handle_run_rollout=None):
+async def run_server_and_client():
     """Start a mock ZMQ server and connected client, tearing both down on exit.
 
-    Args:
-        handle_run_rollout: Optional async callable to override the server's
-            ``handle_run_rollout`` method. Useful for injecting slow or
-            observable handlers in tests.
+    The router's worker spawning is mocked out so no subprocesses are created.
+    Instead, dispatch_request/forward_cancel are replaced with AsyncMock so tests can
+    observe request routing without needing real workers.
     """
-    port = get_free_port_pair()
+    port = get_free_port()
     address = f"tcp://127.0.0.1:{port}"
 
     server = make_mock_server(address)
-    if handle_run_rollout is not None:
-        server.handle_run_rollout = handle_run_rollout  # type: ignore[assignment]
+
+    # Mock out worker lifecycle — we don't want real subprocesses in unit tests
+    server.router.start_workers = MagicMock()
+    server.router.dispatch_request = AsyncMock()
+    server.router.forward_cancel = AsyncMock()
 
     stop_event = asyncio.Event()
     server_loop = asyncio.create_task(server.serve(stop_event=stop_event))
@@ -324,142 +326,83 @@ class TestRetryOnServerError:
             await client.close()
 
 
-class TestTaskCancellation:
-    """Tests that client-side cancellation propagates to the server.
+class TestCancelForwarding:
+    """Tests that client-side cancellation is forwarded through the router.
 
-    The client sends an empty-payload cancel signal over the existing ZMQ
-    wire format, and the server cancels the corresponding asyncio task.
+    With the multi-process architecture, the ZMQEnvServer receives cancel
+    signals from the client and forwards them via ``router.forward_cancel()``.
+    These tests verify the server correctly routes cancels to the router.
     """
 
     @pytest.mark.asyncio
-    async def test_cancelled_client_task_should_cancel_server_task_before_request_processing(
-        self,
-    ):
-        """Cancellation should still propagate before process_request enters its body."""
-        process_request_blocked = asyncio.Event()
-        original_process_request_entered = asyncio.Event()
-        server_task_cancelled = asyncio.Event()
-
+    async def test_cancel_signal_forwarded_to_router(self):
+        """Client cancellation sends empty payload, server calls router.forward_cancel."""
         async with run_server_and_client() as (server, client):
-            original_process_request = server.process_request
-
-            async def delayed_process_request(
-                client_id,
-                request_id_bytes,
-                payload_bytes,
-            ):
-                process_request_blocked.set()
-                try:
-                    await asyncio.Event().wait()
-                    original_process_request_entered.set()
-                    return await original_process_request(
-                        client_id,
-                        request_id_bytes,
-                        payload_bytes,
-                    )
-                except asyncio.CancelledError:
-                    server_task_cancelled.set()
-                    raise
-
-            server.process_request = delayed_process_request  # type: ignore[assignment]
-
+            # Send a request
             client_task = asyncio.create_task(
                 client.send_request(
                     make_rollout_request(), RunRolloutResponse, timeout=30
                 )
             )
 
-            await asyncio.wait_for(process_request_blocked.wait(), timeout=5)
-            assert len(server.request_tasks) == 1
-            assert not original_process_request_entered.is_set()
+            # Wait for dispatch to be called
+            await asyncio.sleep(0.3)
+            assert server.router.dispatch_request.call_count == 1
 
+            # Cancel on the client side — this sends an empty-payload frame
             client_task.cancel()
             with pytest.raises(asyncio.CancelledError):
                 await client_task
 
-            await asyncio.wait_for(server_task_cancelled.wait(), timeout=5)
-            assert not original_process_request_entered.is_set()
+            # Give the cancel signal time to propagate
+            await asyncio.sleep(0.3)
+
+            # The server should have forwarded the cancel to the router
+            assert server.router.forward_cancel.call_count == 1
+            call_args = server.router.forward_cancel.call_args
+            # forward_cancel(request_id, client_id)
+            assert isinstance(call_args[0][0], bytes)  # request_id
+            assert isinstance(call_args[0][1], bytes)  # client_id
 
     @pytest.mark.asyncio
-    async def test_cancelled_client_task_should_cancel_server_task(self):
-        """When the asyncio task awaiting send_request() is cancelled on the
-        client, the corresponding server-side task should also be cancelled
-        via the empty-payload cancel signal.
-        """
-        server_task_started = asyncio.Event()
-        server_task_cancelled = asyncio.Event()
-
-        async def slow_handle_run_rollout(request):
-            server_task_started.set()
-            try:
-                await asyncio.sleep(60)
-                return RunRolloutResponse(output=None)
-            except asyncio.CancelledError:
-                server_task_cancelled.set()
-                raise
-
-        async with run_server_and_client(slow_handle_run_rollout) as (server, client):
-            client_task = asyncio.create_task(
-                client.send_request(
-                    make_rollout_request(), RunRolloutResponse, timeout=30
+    async def test_timeout_sends_cancel_to_router(self):
+        """Client timeout sends cancel signal, server calls router.forward_cancel."""
+        async with run_server_and_client() as (server, client):
+            # Use a short timeout
+            with pytest.raises(TimeoutError):
+                await client.send_request(
+                    make_rollout_request(), RunRolloutResponse, timeout=0.5
                 )
-            )
 
-            # Wait for the server to actually start processing
-            await asyncio.wait_for(server_task_started.wait(), timeout=5)
-            assert len(server.request_tasks) == 1
+            # Give the cancel signal time to propagate
+            await asyncio.sleep(0.3)
 
-            # Cancel on the client side
-            client_task.cancel()
-            with pytest.raises(asyncio.CancelledError):
-                await client_task
-
-            # Give the system time to propagate the cancellation
-            await asyncio.sleep(0.5)
+            # Dispatch should have been called
+            assert server.router.dispatch_request.call_count == 1
 
-            # The server-side task SHOULD have been cancelled.
-            # This fails today because the client never tells the server.
-            assert server_task_cancelled.is_set(), (
-                "Server-side task was NOT cancelled even though the client "
-                "cancelled the request. The server is still consuming resources "
-                "for a request nobody is waiting for."
-            )
+            # The server should have forwarded the cancel to the router
+            assert server.router.forward_cancel.call_count == 1
 
     @pytest.mark.asyncio
-    async def test_client_timeout_should_cancel_server_task(self):
-        """When the client times out waiting for a response, the
-        corresponding server-side task should be cancelled via the
-        empty-payload cancel signal.
-        """
-        server_task_started = asyncio.Event()
-        server_task_cancelled = asyncio.Event()
-
-        async def slow_handle_run_rollout(request):
-            server_task_started.set()
-            try:
-                await asyncio.sleep(60)
-                return RunRolloutResponse(output=None)
-            except asyncio.CancelledError:
-                server_task_cancelled.set()
-                raise
-
-        async with run_server_and_client(slow_handle_run_rollout) as (server, client):
-            # Use a very short timeout so the client gives up quickly
-            with pytest.raises(TimeoutError):
-                await client.send_request(
-                    make_rollout_request(), RunRolloutResponse, timeout=0.5
+    async def test_dispatch_called_with_correct_frames(self):
+        """Requests are dispatched to the router with client_id, request_id, payload."""
+        async with run_server_and_client() as (server, client):
+            client_task = asyncio.create_task(
+                client.send_request(
+                    make_rollout_request(), RunRolloutResponse, timeout=30
                 )
+            )
 
-            # Confirm the server started processing
-            await asyncio.wait_for(server_task_started.wait(), timeout=5)
-            assert len(server.request_tasks) == 1
+            await asyncio.sleep(0.3)
 
-            # Give the system time to propagate
-            await asyncio.sleep(0.5)
+            assert server.router.dispatch_request.call_count == 1
+            call_args = server.router.dispatch_request.call_args
+            client_id, request_id, payload = call_args[0]
+            assert isinstance(client_id, bytes)
+            assert isinstance(request_id, bytes)
+            assert isinstance(payload, bytes)
+            assert len(payload) > 0  # non-empty payload = real request
 
-            # The server task SHOULD have been cancelled after client timeout
-            assert server_task_cancelled.is_set(), (
-                "Server-side task was NOT cancelled after client timeout. "
-                "The server continues processing a request that already "
-                "timed out on the client."
-            )
+            client_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await client_task

{verifiers-0.1.12.dev0 → verifiers-0.1.12.dev2}/tests/test_envs.py

@@ -5,6 +5,12 @@ from pathlib import Path
 import pytest
 import tomllib
 
+# Timeout in seconds for each subprocess step
+INSTALL_TIMEOUT = 600  # 10 minutes for venv creation + package install
+IMPORT_TIMEOUT = 120  # 2 minutes for importing a package
+LOAD_TIMEOUT = 300  # 5 minutes for loading an environment (may download datasets)
+EVAL_TIMEOUT = 600  # 10 minutes for running vf-eval with -n 1 -r 1
+
 SKIPPED_ENVS = [
     # Requires EXA_API_KEY environment variable
     "mcp_search_env",
@@ -99,9 +105,17 @@ def test_env(env_dir: Path, tmp_path_factory: pytest.TempPathFactory):
         f"uv pip install {repo_root.as_posix()} && "
         f"uv pip install {env_dir.absolute().as_posix()}"
     )
-    process = subprocess.run(
-        cmd, shell=True, executable="/bin/bash", capture_output=True, text=True
-    )
+    try:
+        process = subprocess.run(
+            cmd,
+            shell=True,
+            executable="/bin/bash",
+            capture_output=True,
+            text=True,
+            timeout=INSTALL_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired:
+        pytest.fail(f"Timed out after {INSTALL_TIMEOUT}s installing {env_dir.name}")
     assert process.returncode == 0, (
         f"Failed to create virtual environment: {process.stderr}"
     )
@@ -114,25 +128,49 @@ def test_env(env_dir: Path, tmp_path_factory: pytest.TempPathFactory):
 def help_test_can_import_env(tmp_venv_dir: Path, env_dir: Path):
     """Test that the environment can be imported as a package."""
     import_cmd = f"cd {tmp_venv_dir} && source .venv/bin/activate && uv run python -c 'import {env_dir.name}'"
-    process = subprocess.run(
-        import_cmd, shell=True, executable="/bin/bash", capture_output=True, text=True
-    )
+    try:
+        process = subprocess.run(
+            import_cmd,
+            shell=True,
+            executable="/bin/bash",
+            capture_output=True,
+            text=True,
+            timeout=IMPORT_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired:
+        pytest.fail(f"Timed out after {IMPORT_TIMEOUT}s importing {env_dir.name}")
     assert process.returncode == 0, "Failed to import environment"
 
 
 def help_test_can_load_env(tmp_venv_dir: Path, env_dir: Path):
     """Test that the environment can be loaded."""
     load_cmd = f"""cd {tmp_venv_dir} && source .venv/bin/activate && uv run python -c 'import verifiers as vf; vf.load_environment("{env_dir.name}")'"""
-    process = subprocess.run(
-        load_cmd, shell=True, executable="/bin/bash", capture_output=True, text=True
-    )
+    try:
+        process = subprocess.run(
+            load_cmd,
+            shell=True,
+            executable="/bin/bash",
+            capture_output=True,
+            text=True,
+            timeout=LOAD_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired:
+        pytest.fail(f"Timed out after {LOAD_TIMEOUT}s loading {env_dir.name}")
     assert process.returncode == 0, "Failed to load environment"
 
 
 def help_test_can_eval_env(tmp_venv_dir: Path, env_dir: Path):
     """Test that the environment can be run via vf-eval."""
     eval_cmd = f"cd {tmp_venv_dir} && source .venv/bin/activate && uv run vf-eval {env_dir.name} -n 1 -r 1 -t 512"
-    process = subprocess.run(
-        eval_cmd, shell=True, executable="/bin/bash", capture_output=True, text=True
-    )
+    try:
+        process = subprocess.run(
+            eval_cmd,
+            shell=True,
+            executable="/bin/bash",
+            capture_output=True,
+            text=True,
+            timeout=EVAL_TIMEOUT,
+        )
+    except subprocess.TimeoutExpired:
+        pytest.fail(f"Timed out after {EVAL_TIMEOUT}s evaluating {env_dir.name}")
     assert process.returncode == 0, "Failed to evaluate environment"