verifiers 0.4.1.dev49__tar.gz → 0.4.1.dev51__tar.gz

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev49
+Version: 0.4.1.dev51
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/grid.py

@@ -68,10 +68,32 @@ COMPOSE_VMS = ("prime", "modal")
 # TODO: drop each entry once its fix lands.
 CI_MODEL_GAPS = {
     "kimi-code": "Prime chat completions rejects kimi-code's prompt_cache_key",
-    "openclaw": "openclaw cannot load the custom `internal/` model provider",
 }
 
 
+# Harnesses whose program rewrites earlier turns by design, so a rollout can record
+# several branches. Each test names the ones it lets fork. Recheck on version bumps.
+FORKS = {
+    # Acts only through tool calls: it drops a tool-less reply from its history and
+    # resends only its format error.
+    "mini-swe-agent": "https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/agents/default.py#L114",
+    # Moves its runtime-context message to the tail of every request and drops the
+    # earlier one, unless the provider's replay policy is append-only.
+    "openclaw": "https://github.com/openclaw/openclaw/blob/v2026.9.5/src/agents/internal-runtime-context.ts#L402-L408",
+}
+
+
+def assert_one_branch(
+    trace: vf.Trace, harness: vf.HarnessConfig, forks: Sequence[str] = ()
+) -> None:
+    """A rollout records one branch, unless its harness is one of `forks` (see `FORKS`)."""
+    assert set(forks) <= FORKS.keys(), (
+        f"no FORKS reason for {set(forks) - FORKS.keys()}"
+    )
+    if harness.id not in forks:
+        assert trace.num_branches == 1, f"{trace.num_branches} branches"
+
+
 def agent_seats(taskset: str, env_id: str = "") -> list[str]:
     """The agent seats the env declares for `taskset`: `agent` for a plain taskset, its
     own roles for a recipe env."""

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/test_agent_user.py

@@ -3,7 +3,7 @@
 upstream (agent-user-harness), on the runtimes that can switch users."""
 
 import pytest
-from grid import grid, runtimes_grid
+from grid import assert_one_branch, grid, runtimes_grid
 
 
 @pytest.mark.parametrize(
@@ -52,11 +52,6 @@ async def test_agent_user_harness(run, harness, harness_runtime, scripted_model)
         client=scripted_model,
     )
     (trace,) = episode.traces
-    # mini-swe-agent only acts through tool calls: it drops a tool-less reply (the
-    # scripted "OK") from its history and resends only its format error, so the
-    # trace forks by design. Recheck on version bumps:
-    # https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/agents/default.py#L114
-    if harness.id != "mini-swe-agent":
-        assert trace.num_branches == 1
+    assert_one_branch(trace, harness, forks=("mini-swe-agent",))
     assert trace.ok, [error.message for error in trace.errors]
     assert trace.num_turns >= 1

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/test_multi_turn.py

@@ -4,7 +4,7 @@ every other `dialect` a harness config can select. A turn the harness re-sends
 differently, beyond what its `replay_view` declares, forks the trace."""
 
 import pytest
-from grid import dialect_grid, grid, mark
+from grid import assert_one_branch, dialect_grid, grid, mark
 
 MULTI_TURN = [
     *grid(needs=("shell",)),
@@ -28,7 +28,7 @@ async def test_multi_turn(run, harness, harness_runtime):
         max_tokens=8192,
     )
     (trace,) = episode.traces
-    assert trace.num_branches == 1
+    assert_one_branch(trace, harness, forks=("openclaw",))
     assert trace.ok, [error.message for error in trace.errors]
     assert trace.num_turns >= 1  # ran a command, then finished
     assert trace.reward == 1.0

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/test_multi_turn_tool.py

@@ -5,7 +5,7 @@ placements (colocated in the harness's box, in a Compose service beside it, or i
 runtime) on the runtime x runtime grid."""
 
 import pytest
-from grid import RUNTIMES, grid, placements
+from grid import RUNTIMES, assert_one_branch, grid, placements
 
 import verifiers.v1 as vf
 
@@ -22,7 +22,7 @@ async def test_tool(run, harness, harness_runtime):
         taskset_overrides={"task": {"tools": {"colocated": True}}},
     )
     (trace,) = episode.traces
-    assert trace.num_branches == 1
+    assert_one_branch(trace, harness, forks=("openclaw",))
     assert trace.ok, [error.message for error in trace.errors]
     assert trace.num_turns >= 1
     # Reward 1.0 is the proof the tool ran. How a program surfaces an MCP tool to the

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/test_network_policy.py

@@ -4,7 +4,7 @@ agent runs an egress probe under each policy the task data declares
 on prime."""
 
 import pytest
-from grid import grid
+from grid import assert_one_branch, grid
 
 from verifiers.v1.utils.loaders import harness_class
 
@@ -54,12 +54,7 @@ async def test_network_policy(run, harness, harness_runtime, notice):
     assert set(by_name) == set(NOTICES)
     for name, episode in by_name.items():
         [trace] = episode.traces
-        # mini-swe-agent only acts through tool calls: it drops a tool-less reply
-        # (the probe output as a final answer) from its history, so the trace forks
-        # by design. Recheck on version bumps:
-        # https://github.com/SWE-agent/mini-swe-agent/blob/v2.4.6/src/minisweagent/agents/default.py#L114
-        if harness.id != "mini-swe-agent":
-            assert trace.num_branches == 1
+        assert_one_branch(trace, harness, forks=("mini-swe-agent", "openclaw"))
         assert trace.ok, [error.message for error in trace.errors]
         prompt = prompt_text(trace)
         if notice and NOTICES[name]:

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/test_resume_tool.py

@@ -2,7 +2,7 @@
 exchange (echo-acp-resume), across the harness x runtime grid."""
 
 import pytest
-from grid import grid
+from grid import assert_one_branch, grid
 
 
 @pytest.mark.parametrize(
@@ -19,7 +19,7 @@ async def test_resume_tool(run, harness, harness_runtime):
         rollout_timeout=600,
     )
     (trace,) = episode.traces
-    assert trace.num_branches == 1
+    assert_one_branch(trace, harness, forks=("openclaw",))
     assert trace.ok, [error.message for error in trace.errors]
     assert trace.stop_condition == "user_closed"
     assert trace.rewards["resumed"].score == 1.0

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/test_skills.py

@@ -2,7 +2,7 @@
 installs (skill-codeword), across the harness x runtime grid."""
 
 import pytest
-from grid import grid
+from grid import assert_one_branch, grid
 
 
 @pytest.mark.parametrize(
@@ -17,6 +17,6 @@ async def test_skills(run, harness, harness_runtime):
         max_tokens=8192,
     )
     (trace,) = episode.traces
-    assert trace.num_branches == 1
+    assert_one_branch(trace, harness, forks=("openclaw",))
     assert trace.ok, [error.message for error in trace.errors]
     assert trace.reward == 1.0, trace.last_reply

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/tests/integration/test_user_echo.py

@@ -3,7 +3,7 @@ conversation each turn (echo-user-sim), across the harness x runtime grid; and t
 caller drives an agent turn by turn through `agent.interaction`."""
 
 import pytest
-from grid import grid
+from grid import assert_one_branch, grid
 
 import verifiers.v1 as vf
 
@@ -25,7 +25,7 @@ async def test_user(run, harness, harness_runtime):
         max_tokens=8192,
     )
     (trace,) = episode.traces
-    assert trace.num_branches == 1
+    assert_one_branch(trace, harness, forks=("openclaw",))
     assert trace.ok, [error.message for error in trace.errors]
     assert trace.num_turns >= 2  # genuinely multi-turn
     assert trace.stop_condition == "user_closed"  # leaving the interaction ended it

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/verifiers/v1/cli/replay.py

@@ -1,7 +1,8 @@
 """Offline scoring for saved traces.
 
-Replay clears each trace's scores and recomputes everything computable from the saved
-transcript — trace-only handlers plus the layered config's judges. Runtime-requiring
+Replay re-scores each trace with `Task.rescore`: its scores cleared, everything
+computable from the saved transcript recomputed — trace-only handlers plus the layered
+config's judges. Runtime-requiring
 signals don't run offline, so a replay carries offline scores only; the source run keeps
 the runtime-recorded values. Its saved config is the base for replay-specific overrides.
 Multi-agent runs don't support replay: re-scoring is per trace, and the env's
@@ -152,16 +153,13 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
                 st.state, st.detail, st.end = "skipped", "rollout errored", time.time()
             else:
                 st.state = "running"
-                # Clear the recorded scores; only offline-recomputable ones come back.
-                trace.info.pop("judge_calls", None)
-                trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
                 # The declared Task for a rebuilt row, the base Task for the
                 # WireTaskData fallback.
                 task = (task_cls if isinstance(row, data_cls) else Task)(
                     row, config=config.taskset.task
                 )
                 try:
-                    await task.score(trace)
+                    await task.rescore(trace)
                     st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
                 except Exception as exc:
                     st.state, st.detail = "error", type(exc).__name__

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/verifiers/v1/harnesses/openclaw/harness.py

@@ -61,6 +61,14 @@ if (patched !== source) {
 NODE
 """
 
+# OpenClaw rejects a model it cannot resolve as the primary model, which makes setting
+# it in a scratch config a check for whether its catalog knows the model.
+CATALOG_CHECK = r"""
+mkdir -p "$OPENCLAW_STATE_DIR"
+printf '{}' >"$OPENCLAW_CONFIG_PATH"
+exec "$VF_OPENCLAW_BIN" config set agents.defaults.model.primary "$VF_OPENCLAW_MODEL"
+"""
+
 OPENCLAW_COMMAND = [
     "sh",
     "-c",
@@ -130,6 +138,10 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
     SUPPORTS_MCP = True
     SUPPORTS_SKILLS = True
 
+    def __init__(self, config: OpenClawHarnessConfig) -> None:
+        super().__init__(config)
+        self._cataloged: dict[str, bool] = {}
+
     async def setup(self, runtime: Runtime) -> None:
         directory = OPENCLAW_DIR.format(version=self.config.version)
         binary = OPENCLAW_BIN.format(version=self.config.version)
@@ -151,6 +163,23 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
         )
         await super().setup(runtime)
 
+    async def _in_catalog(self, runtime: Runtime, model: str, state_dir: str) -> bool:
+        """Whether OpenClaw's model catalog knows `model`, checked once per model."""
+        if model not in self._cataloged:
+            check_dir = f"{state_dir}/catalog-check"
+            result = await runtime.run(
+                ["sh", "-c", CATALOG_CHECK],
+                {
+                    **self.config.resolved_env,
+                    "OPENCLAW_STATE_DIR": check_dir,
+                    "OPENCLAW_CONFIG_PATH": f"{check_dir}/openclaw.json",
+                    "VF_OPENCLAW_BIN": OPENCLAW_BIN.format(version=self.config.version),
+                    "VF_OPENCLAW_MODEL": json.dumps(model),
+                },
+            )
+            self._cataloged[model] = result.exit_code == 0
+        return self._cataloged[model]
+
     async def prepare_acp(
         self,
         ctx: ModelContext,
@@ -162,10 +191,20 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
         data: TaskData,
     ) -> ACPConfig:
         system_prompt, prompt = self.resolve_prompt(data)
-        provider, _, _ = ctx.model.partition("/")
+        provider, _, model_id = ctx.model.partition("/")
         state_dir = f".vf-openclaw/{trace.id}"
         config_path = f"{state_dir}/openclaw.json"
         skills_dir = f"{state_dir}/skills"
+        provider_config = {
+            "baseUrl": endpoint,
+            "apiKey": "${OPENCLAW_INTERCEPT_KEY}",
+            "request": {"allowPrivateNetwork": True},
+        }
+        if not await self._in_catalog(runtime, ctx.model, state_dir):
+            # Catalog models keep OpenClaw's defaults. Others need a declared model, and
+            # OpenClaw's chat completions default for them rewrites replayed turns.
+            provider_config["api"] = "openai-responses"
+            provider_config["models"] = [{"id": model_id, "name": model_id}]
         config = {
             "gateway": {
                 "mode": "local",
@@ -193,13 +232,7 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
                 "deny": self.config.disabled_tools or [],
             },
             "models": {
-                "providers": {
-                    provider: {
-                        "baseUrl": endpoint,
-                        "apiKey": "${OPENCLAW_INTERCEPT_KEY}",
-                        "request": {"allowPrivateNetwork": True},
-                    }
-                },
+                "providers": {provider: provider_config},
             },
             "mcp": {
                 "servers": {

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev51}/verifiers/v1/task.py

@@ -224,6 +224,15 @@ class Task(Generic[DataT, StateT, ConfigT]):
         as a rollout would."""
         return None
 
+    async def rescore(self, trace: Trace) -> None:
+        """Score a finished trace again, offline: clear its rewards, metrics and judge
+        calls, then run this task's signals that need only the trace."""
+        if trace.has_error:
+            raise ValueError("the rollout errored: its transcript is incomplete")
+        trace.info.pop("judge_calls", None)
+        trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
+        await self.score(trace)
+
     def defer_scoring(self) -> Self:
         """An independent copy whose task signals are deferred.