verifiers 0.1.15.dev2__tar.gz → 0.1.15.dev3__tar.gz

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev2
+Version: 0.1.15.dev3
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/tests/test_interception_utils.py

@@ -261,7 +261,7 @@ async def test_keepalive_write_failure_surfaces_to_state(monkeypatch):
 
     assert isinstance(state["error"], StreamInterrupted)
     msg = str(state["error"])
-    assert "keepalive write failed" in msg
+    assert "Keepalive write failed" in msg
     assert "ConnectionResetError" in msg
 
 
@@ -306,6 +306,6 @@ async def test_non_streaming_response_future_failure_surfaces_to_state(monkeypat
         f"expected InterceptionError, got {type(state.get('error'))}"
     )
     msg = str(state["error"])
-    assert "intercepted request failed" in msg
+    assert "Intercepted request failed" in msg
     assert "RuntimeError" in msg
     assert "vLLM raised" in msg

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/tests/test_v1_harbor_cli.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import json
+import sys
+import types
 from pathlib import Path
 from typing import cast
 
@@ -8,6 +10,7 @@ import pytest
 
 import verifiers.v1 as vf
 from verifiers.v1.packages.harnesses.pi import pi_mcp_json, pi_models_json
+from verifiers.v1.packages.tasksets.harbor import harbor_reward
 from verifiers.v1.utils.program_utils import merge_task_program
 
 
@@ -82,6 +85,76 @@ def test_harbor_taskset_constructs_env_with_opencode(tmp_path: Path) -> None:
     assert "task_dir" not in cast(dict[str, object], env.harness.program)
 
 
+class FakeHarborCommandResult:
+    def __init__(
+        self,
+        *,
+        exit_code: int = 0,
+        stdout: str = "",
+        stderr: str = "",
+    ):
+        self.exit_code = exit_code
+        self.stdout = stdout
+        self.stderr = stderr
+
+
+class FakeHarborSandboxClient:
+    instances: list["FakeHarborSandboxClient"] = []
+
+    def __init__(self):
+        self.execute_commands: list[tuple[str, int | None, str | None]] = []
+        self.background_jobs: list[tuple[str, str, int | None, str | None]] = []
+        type(self).instances.append(self)
+
+    async def upload_file(self, *args: object, **kwargs: object) -> None:
+        _ = args, kwargs
+
+    async def execute_command(
+        self, *args: object, **kwargs: object
+    ) -> FakeHarborCommandResult:
+        command = str(kwargs.get("command") or args[1])
+        timeout = cast(int | None, kwargs.get("timeout"))
+        working_dir = cast(str | None, kwargs.get("working_dir"))
+        self.execute_commands.append((command, timeout, working_dir))
+        if "reward.txt" in command:
+            return FakeHarborCommandResult(stdout="1\n")
+        return FakeHarborCommandResult()
+
+    async def run_background_job(
+        self, *args: object, **kwargs: object
+    ) -> FakeHarborCommandResult:
+        sandbox_id = str(kwargs.get("sandbox_id") or args[0])
+        command = str(kwargs.get("command") or args[1])
+        timeout = cast(int | None, kwargs.get("timeout"))
+        working_dir = cast(str | None, kwargs.get("working_dir"))
+        self.background_jobs.append((sandbox_id, command, timeout, working_dir))
+        return FakeHarborCommandResult(stdout="tests passed")
+
+    async def aclose(self) -> None:
+        pass
+
+
+@pytest.mark.asyncio
+async def test_harbor_reward_uses_background_job_for_tests(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    task_dir = write_harbor_task(tmp_path)
+    fake_module = types.ModuleType("prime_sandboxes")
+    fake_module.AsyncSandboxClient = FakeHarborSandboxClient
+    monkeypatch.setitem(sys.modules, "prime_sandboxes", fake_module)
+    FakeHarborSandboxClient.instances = []
+
+    reward = await harbor_reward(
+        {"harbor": {"task_dir": str(task_dir), "test_timeout": 120}},
+        {"sandbox_id": "sbx-1"},
+    )
+
+    client = FakeHarborSandboxClient.instances[0]
+    assert reward == 1.0
+    assert client.background_jobs == [("sbx-1", "bash test.sh", 120, "/tests")]
+    assert ("bash test.sh", 120, "/tests") not in client.execute_commands
+
+
 def test_packaged_harbor_and_opencode_imports_are_reexported() -> None:
     from verifiers.v1.packages.harnesses import OpenCode, OpenCodeConfig, Pi
     from verifiers.v1.packages.tasksets import HarborTaskset

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/tests/test_v1_runtime_lifecycle.py

@@ -109,6 +109,7 @@ class FakeSandboxClient:
     created: list[str] = []
     deleted: list[str] = []
     commands: list[tuple[str, str]] = []
+    background_jobs: list[tuple[str, str, int | None, str | None]] = []
     uploads: list[tuple[str, str, bytes]] = []
 
     @classmethod
@@ -116,6 +117,7 @@ class FakeSandboxClient:
         cls.created = []
         cls.deleted = []
         cls.commands = []
+        cls.background_jobs = []
         cls.uploads = []
 
     async def create(self, request: FakeCreateSandboxRequest) -> FakeSandboxResult:
@@ -135,6 +137,17 @@ class FakeSandboxClient:
         type(self).commands.append((sandbox_id, command))
         return FakeCommandResult()
 
+    async def run_background_job(
+        self, *args: object, **kwargs: object
+    ) -> FakeCommandResult:
+        sandbox_id = str(kwargs.get("sandbox_id") or args[0])
+        command = str(kwargs.get("command") or args[1])
+        timeout = cast(int | None, kwargs.get("timeout"))
+        working_dir = cast(str | None, kwargs.get("working_dir"))
+        type(self).commands.append((sandbox_id, command))
+        type(self).background_jobs.append((sandbox_id, command, timeout, working_dir))
+        return FakeCommandResult()
+
     async def upload_bytes(self, *args: object, **kwargs: object) -> None:
         sandbox_id = str(kwargs.get("sandbox_id") or args[0])
         path = str(kwargs.get("file_path") or kwargs.get("path") or args[1])
@@ -1193,6 +1206,26 @@ async def test_sandbox_state_input_upload_runs_after_rollout_setup(
     assert uploads["/tmp/vf_state_in.json"]["state_input_setup"] is True
 
 
+@pytest.mark.asyncio
+async def test_task_command_uses_background_job(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    install_fake_sandboxes(monkeypatch)
+    install_fake_endpoint_tunnel(monkeypatch)
+
+    harness = vf.CLIHarness(command=["sleep", "120"], sandbox=True)
+    task = vf.Task(
+        {
+            "prompt": [{"role": "user", "content": "hi"}],
+            "sandbox": {"command_timeout": 120},
+        }
+    ).freeze()
+
+    await harness.run(task)
+
+    assert ("sbx-1", "sleep 120", 120, "/app") in FakeSandboxClient.background_jobs
+
+
 @pytest.mark.asyncio
 async def test_program_tools_mcp_setup_accepts_config_ref_mappings(
     monkeypatch: pytest.MonkeyPatch,

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev2"
+__version__ = "0.1.15.dev3"
 
 import importlib
 import os

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/verifiers/utils/interception_utils.py

@@ -173,12 +173,20 @@ class InterceptionServer:
         """Attach `error` to the rollout's state if one is registered and
         unset. First error wins — later failures (e.g. the downstream
         `response_future` raising too) should not clobber the original cause.
+
+        Also skip when the rollout loop has already finalized via a clean
+        stop condition (e.g. ``state["prompt_too_long"]`` from an
+        ``OverlongPromptError``). Tail-end failures that happen after
+        that — e.g. ``write_eof`` to an agent that has already exited —
+        are consequences of the termination, not new infra problems, and
+        must not be surfaced as a spurious ``InterceptionError`` /
+        ``StreamInterrupted`` alongside the real stop signal.
         """
         context = self.active_rollouts.get(rollout_id)
         if context is None:
             return
         state = context.get("state")
-        if state is None or state.get("error"):
+        if state is None or state.get("error") or state.get("prompt_too_long"):
             return
         state["error"] = error
 
@@ -295,7 +303,7 @@ class InterceptionServer:
                 self._set_rollout_error(
                     rollout_id,
                     InterceptionError(
-                        f"intercepted request failed: {type(e).__name__}: {e}"
+                        f"Intercepted request failed: {type(e).__name__}: {e}"
                     ),
                 )
                 return web.json_response({"error": str(e)}, status=500)
@@ -433,7 +441,7 @@ class InterceptionServer:
             )
             self._set_rollout_error(
                 rollout_id,
-                StreamInterrupted(f"prepare failed: {type(e).__name__}: {e}"),
+                StreamInterrupted(f"Prepare failed: {type(e).__name__}: {e}"),
             )
             return response
         # Reuse one get() task across keepalive cycles; asyncio.wait_for on
@@ -460,7 +468,7 @@ class InterceptionServer:
                         self._set_rollout_error(
                             rollout_id,
                             StreamInterrupted(
-                                f"keepalive write failed after {print_time(waited_s)}: "
+                                f"Keepalive write failed after {print_time(waited_s)}: "
                                 f"{type(e).__name__}: {e}"
                             ),
                         )
@@ -490,7 +498,7 @@ class InterceptionServer:
             self._set_rollout_error(
                 rollout_id,
                 StreamInterrupted(
-                    f"stream write failed after {print_time(waited_s)}: "
+                    f"Stream write failed after {print_time(waited_s)}: "
                     f"{type(e).__name__}: {e}"
                 ),
             )
@@ -510,7 +518,7 @@ class InterceptionServer:
             self._set_rollout_error(
                 rollout_id,
                 StreamInterrupted(
-                    f"streaming response_future failed: {type(e).__name__}: {e}"
+                    f"Streaming response_future failed: {type(e).__name__}: {e}"
                 ),
             )
 
@@ -527,7 +535,7 @@ class InterceptionServer:
             self._set_rollout_error(
                 rollout_id,
                 StreamInterrupted(
-                    f"write_eof failed after {print_time(waited_s)}: "
+                    f"Write EOF failed after {print_time(waited_s)}: "
                     f"{type(e).__name__}: {e}"
                 ),
             )

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/verifiers/v1/packages/tasksets/harbor.py

@@ -333,11 +333,12 @@ async def harbor_reward(task, state) -> float:
     client = AsyncSandboxClient()
     try:
         await upload_harbor_tests(client, sandbox_id, task_dir)
-        result = await client.execute_command(
+        test_timeout = int(parse_number(harbor.get("test_timeout"), 900))
+        result = await client.run_background_job(
             sandbox_id=sandbox_id,
             command="bash test.sh",
             working_dir="/tests",
-            timeout=int(parse_number(harbor.get("test_timeout"), 900)),
+            timeout=test_timeout,
         )
         state["harbor_tests"] = {
             "returncode": result.exit_code,

{verifiers-0.1.15.dev2 → verifiers-0.1.15.dev3}/verifiers/v1/utils/sandbox_utils.py

@@ -262,12 +262,13 @@ async def run_sandbox_command(
             )
         argv = await command_argv(program, task, state, runtime)
         env = await command_env(program, task, state, runtime, include_base=False)
-        result = await lease.client.execute_command(
-            lease.id,
-            shlex.join(argv),
+        command = shlex.join(argv)
+        command_timeout = cast(int | None, sandbox_config.get("command_timeout"))
+        result = await lease.run_background_job(
+            command,
+            timeout=command_timeout,
             working_dir=workdir,
             env=env,
-            timeout=cast(int | None, sandbox_config.get("command_timeout")),
         )
         state["command"] = {
             "argv": argv,