verifiers 0.1.15.dev1__tar.gz → 0.1.15.dev3__tar.gz

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev1
+Version: 0.1.15.dev3
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/tests/test_interception_utils.py

@@ -261,7 +261,7 @@ async def test_keepalive_write_failure_surfaces_to_state(monkeypatch):
 
     assert isinstance(state["error"], StreamInterrupted)
     msg = str(state["error"])
-    assert "keepalive write failed" in msg
+    assert "Keepalive write failed" in msg
     assert "ConnectionResetError" in msg
 
 
@@ -306,6 +306,6 @@ async def test_non_streaming_response_future_failure_surfaces_to_state(monkeypat
         f"expected InterceptionError, got {type(state.get('error'))}"
     )
     msg = str(state["error"])
-    assert "intercepted request failed" in msg
+    assert "Intercepted request failed" in msg
     assert "RuntimeError" in msg
     assert "vLLM raised" in msg

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/tests/test_v1_harbor_cli.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 import json
+import sys
+import types
 from pathlib import Path
 from typing import cast
 
@@ -8,6 +10,7 @@ import pytest
 
 import verifiers.v1 as vf
 from verifiers.v1.packages.harnesses.pi import pi_mcp_json, pi_models_json
+from verifiers.v1.packages.tasksets.harbor import harbor_reward
 from verifiers.v1.utils.program_utils import merge_task_program
 
 
@@ -82,6 +85,76 @@ def test_harbor_taskset_constructs_env_with_opencode(tmp_path: Path) -> None:
     assert "task_dir" not in cast(dict[str, object], env.harness.program)
 
 
+class FakeHarborCommandResult:
+    def __init__(
+        self,
+        *,
+        exit_code: int = 0,
+        stdout: str = "",
+        stderr: str = "",
+    ):
+        self.exit_code = exit_code
+        self.stdout = stdout
+        self.stderr = stderr
+
+
+class FakeHarborSandboxClient:
+    instances: list["FakeHarborSandboxClient"] = []
+
+    def __init__(self):
+        self.execute_commands: list[tuple[str, int | None, str | None]] = []
+        self.background_jobs: list[tuple[str, str, int | None, str | None]] = []
+        type(self).instances.append(self)
+
+    async def upload_file(self, *args: object, **kwargs: object) -> None:
+        _ = args, kwargs
+
+    async def execute_command(
+        self, *args: object, **kwargs: object
+    ) -> FakeHarborCommandResult:
+        command = str(kwargs.get("command") or args[1])
+        timeout = cast(int | None, kwargs.get("timeout"))
+        working_dir = cast(str | None, kwargs.get("working_dir"))
+        self.execute_commands.append((command, timeout, working_dir))
+        if "reward.txt" in command:
+            return FakeHarborCommandResult(stdout="1\n")
+        return FakeHarborCommandResult()
+
+    async def run_background_job(
+        self, *args: object, **kwargs: object
+    ) -> FakeHarborCommandResult:
+        sandbox_id = str(kwargs.get("sandbox_id") or args[0])
+        command = str(kwargs.get("command") or args[1])
+        timeout = cast(int | None, kwargs.get("timeout"))
+        working_dir = cast(str | None, kwargs.get("working_dir"))
+        self.background_jobs.append((sandbox_id, command, timeout, working_dir))
+        return FakeHarborCommandResult(stdout="tests passed")
+
+    async def aclose(self) -> None:
+        pass
+
+
+@pytest.mark.asyncio
+async def test_harbor_reward_uses_background_job_for_tests(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    task_dir = write_harbor_task(tmp_path)
+    fake_module = types.ModuleType("prime_sandboxes")
+    fake_module.AsyncSandboxClient = FakeHarborSandboxClient
+    monkeypatch.setitem(sys.modules, "prime_sandboxes", fake_module)
+    FakeHarborSandboxClient.instances = []
+
+    reward = await harbor_reward(
+        {"harbor": {"task_dir": str(task_dir), "test_timeout": 120}},
+        {"sandbox_id": "sbx-1"},
+    )
+
+    client = FakeHarborSandboxClient.instances[0]
+    assert reward == 1.0
+    assert client.background_jobs == [("sbx-1", "bash test.sh", 120, "/tests")]
+    assert ("bash test.sh", 120, "/tests") not in client.execute_commands
+
+
 def test_packaged_harbor_and_opencode_imports_are_reexported() -> None:
     from verifiers.v1.packages.harnesses import OpenCode, OpenCodeConfig, Pi
     from verifiers.v1.packages.tasksets import HarborTaskset

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/tests/test_v1_runtime_lifecycle.py

@@ -109,6 +109,7 @@ class FakeSandboxClient:
     created: list[str] = []
     deleted: list[str] = []
     commands: list[tuple[str, str]] = []
+    background_jobs: list[tuple[str, str, int | None, str | None]] = []
     uploads: list[tuple[str, str, bytes]] = []
 
     @classmethod
@@ -116,6 +117,7 @@ class FakeSandboxClient:
         cls.created = []
         cls.deleted = []
         cls.commands = []
+        cls.background_jobs = []
         cls.uploads = []
 
     async def create(self, request: FakeCreateSandboxRequest) -> FakeSandboxResult:
@@ -135,6 +137,17 @@ class FakeSandboxClient:
         type(self).commands.append((sandbox_id, command))
         return FakeCommandResult()
 
+    async def run_background_job(
+        self, *args: object, **kwargs: object
+    ) -> FakeCommandResult:
+        sandbox_id = str(kwargs.get("sandbox_id") or args[0])
+        command = str(kwargs.get("command") or args[1])
+        timeout = cast(int | None, kwargs.get("timeout"))
+        working_dir = cast(str | None, kwargs.get("working_dir"))
+        type(self).commands.append((sandbox_id, command))
+        type(self).background_jobs.append((sandbox_id, command, timeout, working_dir))
+        return FakeCommandResult()
+
     async def upload_bytes(self, *args: object, **kwargs: object) -> None:
         sandbox_id = str(kwargs.get("sandbox_id") or args[0])
         path = str(kwargs.get("file_path") or kwargs.get("path") or args[1])
@@ -1193,6 +1206,26 @@ async def test_sandbox_state_input_upload_runs_after_rollout_setup(
     assert uploads["/tmp/vf_state_in.json"]["state_input_setup"] is True
 
 
+@pytest.mark.asyncio
+async def test_task_command_uses_background_job(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    install_fake_sandboxes(monkeypatch)
+    install_fake_endpoint_tunnel(monkeypatch)
+
+    harness = vf.CLIHarness(command=["sleep", "120"], sandbox=True)
+    task = vf.Task(
+        {
+            "prompt": [{"role": "user", "content": "hi"}],
+            "sandbox": {"command_timeout": 120},
+        }
+    ).freeze()
+
+    await harness.run(task)
+
+    assert ("sbx-1", "sleep 120", 120, "/app") in FakeSandboxClient.background_jobs
+
+
 @pytest.mark.asyncio
 async def test_program_tools_mcp_setup_accepts_config_ref_mappings(
     monkeypatch: pytest.MonkeyPatch,

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev1"
+__version__ = "0.1.15.dev3"
 
 import importlib
 import os

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/verifiers/envs/experimental/composable/harnesses/opencode.py

@@ -113,6 +113,9 @@ def build_opencode_config(
     provider_timeout_ms: int = 3_600_000,
 ) -> str:
     """Generate opencode.json config content."""
+    agent_config: dict[str, object] = {
+        "title": {"disable": True},
+    }
     config: dict = {
         "${SCHEMA_DOLLAR}schema": "https://opencode.ai/config.json",
         "provider": {
@@ -134,12 +137,10 @@ def build_opencode_config(
             }
         },
         "model": model_id,
-        # Without small_model, title/summary sub-agents fall through to
-        # opencode/gpt-5-nano (free hosted Cloudflare tier) and rate-limit
-        # under concurrent rollouts, which silently blocks the main agent
-        # before it issues any LLM call. See OpenCode provider.ts
-        # getSmallModel() priority chain. Pin to the same intercepted model.
+        # Keep the small-model pin to avoid falling back to the default small
+        # model and hitting rate limits; disable title calls below.
         "small_model": model_id,
+        "agent": agent_config,
     }
 
     if disable_compaction:
@@ -151,7 +152,7 @@ def build_opencode_config(
     if disabled_tools:
         agent_build["tools"] = {tool: False for tool in disabled_tools}
     if agent_build:
-        config["agent"] = {"build": agent_build}
+        agent_config["build"] = agent_build
 
     return json.dumps(config, indent=2)
 

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/verifiers/envs/experimental/opencode_env.py

@@ -362,6 +362,9 @@ class OpenCodeEnv(CliAgentEnv):
         enable_interleaved: bool = True,
     ) -> str:
         """Build OpenCode config."""
+        agent_config: dict[str, object] = {
+            "title": {"disable": True},
+        }
         config: dict = {
             "${SCHEMA_DOLLAR}schema": "https://opencode.ai/config.json",
             "provider": {
@@ -388,9 +391,10 @@ class OpenCodeEnv(CliAgentEnv):
                 }
             },
             "model": "$OPENAI_MODEL",
-            # Keep title/summary sub-agents on the intercepted provider instead
-            # of falling back to OpenCode's hosted free small model.
+            # Keep the small-model pin to avoid falling back to the default small
+            # model and hitting rate limits; disable title calls below.
             "small_model": "$OPENAI_MODEL",
+            "agent": agent_config,
         }
 
         if disable_compaction:
@@ -402,7 +406,7 @@ class OpenCodeEnv(CliAgentEnv):
                 build_config["prompt"] = "{file:" + system_prompt_path + "}"
             if disabled_tools:
                 build_config["tools"] = {tool: False for tool in disabled_tools}
-            config["agent"] = {"build": build_config}
+            agent_config["build"] = build_config
 
         return json.dumps(config, indent=2)
 

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/verifiers/utils/interception_utils.py

@@ -173,12 +173,20 @@ class InterceptionServer:
         """Attach `error` to the rollout's state if one is registered and
         unset. First error wins — later failures (e.g. the downstream
         `response_future` raising too) should not clobber the original cause.
+
+        Also skip when the rollout loop has already finalized via a clean
+        stop condition (e.g. ``state["prompt_too_long"]`` from an
+        ``OverlongPromptError``). Tail-end failures that happen after
+        that — e.g. ``write_eof`` to an agent that has already exited —
+        are consequences of the termination, not new infra problems, and
+        must not be surfaced as a spurious ``InterceptionError`` /
+        ``StreamInterrupted`` alongside the real stop signal.
         """
         context = self.active_rollouts.get(rollout_id)
         if context is None:
             return
         state = context.get("state")
-        if state is None or state.get("error"):
+        if state is None or state.get("error") or state.get("prompt_too_long"):
             return
         state["error"] = error
 
@@ -295,7 +303,7 @@ class InterceptionServer:
                 self._set_rollout_error(
                     rollout_id,
                     InterceptionError(
-                        f"intercepted request failed: {type(e).__name__}: {e}"
+                        f"Intercepted request failed: {type(e).__name__}: {e}"
                     ),
                 )
                 return web.json_response({"error": str(e)}, status=500)
@@ -433,7 +441,7 @@ class InterceptionServer:
             )
             self._set_rollout_error(
                 rollout_id,
-                StreamInterrupted(f"prepare failed: {type(e).__name__}: {e}"),
+                StreamInterrupted(f"Prepare failed: {type(e).__name__}: {e}"),
             )
             return response
         # Reuse one get() task across keepalive cycles; asyncio.wait_for on
@@ -460,7 +468,7 @@ class InterceptionServer:
                         self._set_rollout_error(
                             rollout_id,
                             StreamInterrupted(
-                                f"keepalive write failed after {print_time(waited_s)}: "
+                                f"Keepalive write failed after {print_time(waited_s)}: "
                                 f"{type(e).__name__}: {e}"
                             ),
                         )
@@ -490,7 +498,7 @@ class InterceptionServer:
             self._set_rollout_error(
                 rollout_id,
                 StreamInterrupted(
-                    f"stream write failed after {print_time(waited_s)}: "
+                    f"Stream write failed after {print_time(waited_s)}: "
                     f"{type(e).__name__}: {e}"
                 ),
             )
@@ -510,7 +518,7 @@ class InterceptionServer:
             self._set_rollout_error(
                 rollout_id,
                 StreamInterrupted(
-                    f"streaming response_future failed: {type(e).__name__}: {e}"
+                    f"Streaming response_future failed: {type(e).__name__}: {e}"
                 ),
             )
 
@@ -527,7 +535,7 @@ class InterceptionServer:
             self._set_rollout_error(
                 rollout_id,
                 StreamInterrupted(
-                    f"write_eof failed after {print_time(waited_s)}: "
+                    f"Write EOF failed after {print_time(waited_s)}: "
                     f"{type(e).__name__}: {e}"
                 ),
             )

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/verifiers/v1/packages/harnesses/opencode.py

@@ -198,6 +198,9 @@ def build_opencode_config(
     disable_compaction: bool,
     provider_timeout_ms: int,
 ) -> str:
+    agent_config: dict[str, object] = {
+        "title": {"disable": True},
+    }
     config: dict[str, object] = {
         "${SCHEMA_DOLLAR}schema": "https://opencode.ai/config.json",
         "provider": {
@@ -218,9 +221,10 @@ def build_opencode_config(
             }
         },
         "model": "intercepted/model",
-        # Keep title/summary sub-agents on the intercepted provider instead
-        # of falling back to OpenCode's hosted free small model.
+        # Keep the small-model pin to avoid falling back to the default small
+        # model and hitting rate limits; disable title calls below.
         "small_model": "intercepted/model",
+        "agent": agent_config,
         "mcp": {
             "verifiers-tools": {
                 "type": "local",
@@ -237,7 +241,7 @@ def build_opencode_config(
     if disabled_tools:
         build_config["tools"] = {tool: False for tool in disabled_tools}
     if build_config:
-        config["agent"] = {"build": build_config}
+        agent_config["build"] = build_config
     return json.dumps(config, indent=2)
 
 

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/verifiers/v1/packages/tasksets/harbor.py

@@ -333,11 +333,12 @@ async def harbor_reward(task, state) -> float:
     client = AsyncSandboxClient()
     try:
         await upload_harbor_tests(client, sandbox_id, task_dir)
-        result = await client.execute_command(
+        test_timeout = int(parse_number(harbor.get("test_timeout"), 900))
+        result = await client.run_background_job(
             sandbox_id=sandbox_id,
             command="bash test.sh",
             working_dir="/tests",
-            timeout=int(parse_number(harbor.get("test_timeout"), 900)),
+            timeout=test_timeout,
         )
         state["harbor_tests"] = {
             "returncode": result.exit_code,

{verifiers-0.1.15.dev1 → verifiers-0.1.15.dev3}/verifiers/v1/utils/sandbox_utils.py

@@ -262,12 +262,13 @@ async def run_sandbox_command(
             )
         argv = await command_argv(program, task, state, runtime)
         env = await command_env(program, task, state, runtime, include_base=False)
-        result = await lease.client.execute_command(
-            lease.id,
-            shlex.join(argv),
+        command = shlex.join(argv)
+        command_timeout = cast(int | None, sandbox_config.get("command_timeout"))
+        result = await lease.run_background_job(
+            command,
+            timeout=command_timeout,
             working_dir=workdir,
             env=env,
-            timeout=cast(int | None, sandbox_config.get("command_timeout")),
         )
         state["command"] = {
             "argv": argv,