verifiers 0.2.2.dev96__tar.gz → 0.2.2.dev112__tar.gz

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev96
+Version: 0.2.2.dev112
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -39,7 +39,7 @@ Requires-Dist: prime-sandboxes>=0.2.35
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
-Requires-Dist: renderers>=0.1.9.dev9
+Requires-Dist: renderers>=0.1.9
 Requires-Dist: requests
 Requires-Dist: rich>=11.0.0
 Requires-Dist: setproctitle>=1.3.0
@@ -54,6 +54,8 @@ Provides-Extra: harbor
 Requires-Dist: harbor==0.20.0; (python_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
 Requires-Dist: modal>=1.4.0; extra == 'modal'
+Provides-Extra: nemo-gym
+Requires-Dist: nemo-gym==0.4.0; (python_version >= '3.12') and extra == 'nemo-gym'
 Provides-Extra: notebook
 Requires-Dist: nest-asyncio>=1.6.0; extra == 'notebook'
 Provides-Extra: openenv

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/pyproject.toml

@@ -51,7 +51,7 @@ dependencies = [
     "uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",
-    "renderers>=0.1.9.dev9",
+    "renderers>=0.1.9",
     "typing_extensions>=4.12.2",
 ]
 
@@ -107,12 +107,43 @@ notebook = [
 openenv = [
     "openenv>=0.4.1",
 ]
+nemo-gym = [
+    # NeMo Gym currently publishes Python 3.12+ packages; keep core verifiers installable on 3.11.
+    "nemo-gym==0.4.0; python_version >= '3.12'",
+]
 [tool.uv]
 preview = true
 required-version = ">=0.11.1"
 exclude-newer = "7 days"
 default-groups = ["dev", "examples"]
 
+# NeMo Gym declares its full platform stack; this integration only starts resource servers.
+[[tool.uv.dependency-metadata]]
+name = "nemo-gym"
+version = "0.4.0"
+requires-python = ">=3.12"
+requires-dist = [
+    "aiohttp>=3.14.1",
+    "fastapi",
+    "gprof2dot",
+    "hydra-core",
+    "itsdangerous",
+    "mcp>=1.27,<2",
+    "omegaconf",
+    "openai>=2.9",
+    "orjson",
+    "pandas",
+    "pydantic",
+    "pydot",
+    "ray>=2.55.1",
+    "requests",
+    "rich",
+    "typing-extensions",
+    "uvicorn",
+    "wandb",
+    "yappi",
+]
+
 [[tool.uv.index]]
 name = "pypi"
 url = "https://pypi.org/simple"

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/tests/v1/conftest.py

@@ -164,7 +164,10 @@ def _eval_config(
         seat_cfg.setdefault("max_output_tokens", max_tokens)
         seat_cfg.setdefault("timeout", {"rollout": rollout_timeout, "scoring": 60})
         # Flake resilience: retries are per-agent now (flat RetryConfig).
-        seat_cfg.setdefault("retries", {"max_retries": 2, "include": ["ProviderError"]})
+        seat_cfg.setdefault(
+            "retries",
+            {"max_retries": 2, "include": ["ProviderError", "HarnessError"]},
+        )
     return EvalConfig(
         env={
             "taskset": taskset_cfg,

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/tests/v1/fixtures/echo_tool_v1.py

@@ -11,6 +11,7 @@ so it would also serve taskset-scoped (`Taskset.toolsets`).
 """
 
 import verifiers.v1 as vf
+from verifiers.v1.types import content_text
 
 PHRASE = "hello world"
 ECHO_TOKEN = "ok-7f3"  # the tool stamps this; only a real tool call can surface it
@@ -36,11 +37,9 @@ class EchoToolTask(vf.Task[vf.TaskData, vf.State, EchoToolTaskConfig]):
 
     @vf.reward(weight=1.0)
     async def echoed(self, trace: vf.Trace) -> float:
-        # The stamped token surfaces in an ASSISTANT message only if the model called
-        # the MCP tool and relayed its result — wherever in the exchange that happened
-        # (in a conversation the last turn may be a closing pleasantry).
-        replies = ((m.content or "").lower() for m in trace.assistant_messages)
-        return float(any(PHRASE in r and ECHO_TOKEN in r for r in replies))
+        # A stamped TOOL result proves the tool really ran with the phrase.
+        results = (content_text(m.content).lower() for m in trace.tool_messages)
+        return float(any(PHRASE in r and ECHO_TOKEN in r for r in results))
 
 
 class EchoToolConfig(vf.TasksetConfig):

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/tests/v1/test_envs.py

@@ -17,10 +17,8 @@ EVAL_TIMEOUT = 600  # 10 minutes for a capped eval (-n 1 -r 1)
 
 ENVIRONMENTS = Path(__file__).parent.parent.parent / "environments"
 
-# v1 tasksets that can't run a plain-CI smoke eval — e.g. they need a docker/prime runtime or
-# clone a corpus CI can't read. Empty: the SWE/container and corpus tasksets live in
-# research-environments now.
-SKIP_EVAL: set[str] = set()
+# V1 tasksets that aren't part of the default CI install.
+SKIP_EVAL = {"nemo_gym_weather_v1"}
 
 # Per-run caps are seat fields; recipe envs name their own seats.
 SEATS: dict[str, tuple[str, ...]] = {

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/acp/__init__.py

@@ -121,7 +121,9 @@ class ACP:
             "allow_empty_tool_reply": allow_empty_tool_reply,
         }
         program = await runtime.prepare_uv_script(
-            ACP_SOURCE, {**env, "UV_FROZEN": "false"}
+            ACP_SOURCE,
+            {**env, "UV_FROZEN": "false"},
+            activate=False,
         )
         directory = f".vf-acp-{secrets.token_hex(8)}"
         created = await runtime.run(["mkdir", "-m", "700", directory], {})
@@ -198,7 +200,9 @@ class ACPHarnessSession(HarnessSession):
     async def _start(self) -> None:
         self._stderr_tail.clear()
         program = await self.runtime.prepare_uv_script(
-            ACP_SOURCE, {**self.env, "UV_FROZEN": "false"}
+            ACP_SOURCE,
+            {**self.env, "UV_FROZEN": "false"},
+            activate=False,
         )
         process = await self.runtime.open_process([*program, "stream"], self.env)
         self._process = process

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/clients/train.py

@@ -12,6 +12,7 @@ from typing import Any, ClassVar, TypeVar
 from openai import OpenAIError
 from renderers import OverlongPromptError as RendererOverlongPromptError
 from renderers import RenderedTokens, Renderer, RendererConfig
+from renderers.base import ToolCallParseStatus
 
 from verifiers.v1.clients.base import build_async_openai
 from verifiers.v1.clients.client import SESSION_ID_HEADER, Client
@@ -115,6 +116,8 @@ def response_from_generate(
         )
         for i, tc in enumerate(result.get("tool_calls") or [])
         if getattr(tc, "name", None)
+        # TODO: we need a better way for renderers to expose this
+        and getattr(tc, "status", None) != ToolCallParseStatus.UNKNOWN_TOOL
     ] or None
     prompt_ids = result.get("prompt_ids") or []
     completion_ids = result.get("completion_ids") or []
@@ -296,8 +299,9 @@ class ElasticRendererPool:
 class TrainClient(Client):
     """Renders prompts to token ids and calls a vLLM `/inference/v1/generate` engine.
 
-    One client per rollout: it owns its engine connection and takes a slot on the shared
-    `ElasticRendererPool` for each turn."""
+    Owned by the interception server and shared by the rollouts it multiplexes: they reuse
+    its engine connection pool, and each turn takes a slot on the shared
+    `ElasticRendererPool`."""
 
     def __init__(self, config: TrainClientConfig) -> None:
         self.config = config

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/configs/client.py

@@ -1,8 +1,9 @@
 """Client configs: describe an OpenAI-compatible endpoint.
 
 A `BaseClientConfig` is an OpenAI-compatible endpoint (base_url + API-key env var
-+ extra headers); `clients.resolve_client` turns one into a live `Client`, and every
-rollout does so for itself. The default Prime endpoint, API key, and team fall back to
++ extra headers); `clients.resolve_client` turns one into a live `Client` — the
+interception server builds one per distinct config and shares it across the rollouts
+it multiplexes. The default Prime endpoint, API key, and team fall back to
 the active Prime CLI config, so direct `uv run eval` calls behave like `prime eval`.
 Both the eval entrypoint (its model client) and in-env LLM calls (e.g. a judge reward)
 build clients from these. `ClientConfig` is the CLI-selectable discriminated union

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/dialects/chat.py

@@ -141,7 +141,11 @@ def _content_to_wire(content):
 
 def message_to_wire(message: Message) -> dict:
     if message.role == "assistant":
-        wire: dict = {"role": "assistant", "content": message.content}
+        # Strict providers reject `content: null` without tool calls.
+        content = message.content
+        if content is None and not message.tool_calls:
+            content = ""
+        wire: dict = {"role": "assistant", "content": content}
         if message.provider_state:
             wire["reasoning_details"] = message.provider_state
         elif message.reasoning_content is not None:

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/envs/agentic_judge/env.py

@@ -309,8 +309,6 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
 
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
         by_agent = {t.agent.name: t for t in episode.traces}
-        if "judge" not in by_agent:
-            return
         solution, verdict = by_agent["solver"], by_agent["judge"]
         verdicts = RubricVerdicts.model_validate(verdict.info.get("verdict")).verdicts
         criteria = self.config.task.criteria()
@@ -332,6 +330,10 @@ class SharedAgenticJudgeEnv(AgenticJudgeEnv):
     async def run(self, task: vf.Task, agents: vf.Agents) -> None:
         async with agents.solver.provision(task) as box:
             solution = await agents.solver.run(task, runtime=box)
+            if not solution.ok:
+                raise RuntimeError(
+                    "the solver's rollout failed, so the judge never ran"
+                )
             judge_task = JudgeTask.from_trace(solution, self.config.task)
             await agents.judge.run(judge_task, runtime=box)
 
@@ -342,7 +344,7 @@ class IsolatedAgenticJudgeEnv(AgenticJudgeEnv):
     async def run(self, task: vf.Task, agents: vf.Agents) -> None:
         solution = await agents.solver.run(task)
         if not solution.ok:
-            return
+            raise RuntimeError("the solver's rollout failed, so the judge never ran")
         await agents.judge.run(
             JudgeTask.from_trace(solution, self.config.task, share_runtime=False)
         )

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/envs/user_sim/env.py

@@ -21,13 +21,14 @@ from pydantic import Field
 
 import verifiers.v1 as vf
 
-PERSONA = """You are role-playing a USER talking to an AI assistant. This is your situation and goal:
+PERSONA = """You are role-playing a USER talking to an AI assistant. This is your situation — what you want the ASSISTANT to do for you:
 
 {scenario}
 
 Rules:
 - Open the conversation with your request, in your own words.
 - Stay in character: short, natural user messages; never act as the assistant.
+- The assistant does the work, not you: never perform the task yourself — any tool call, answer, or output format it asks for must come from the assistant; ask for it instead.
 - Reveal details only when asked, as a real user would.
 - When the assistant has fully met your goal — or you are convinced it cannot — reply with exactly {done} and nothing else."""
 

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/gepa/runner.py

@@ -73,7 +73,7 @@ def run_gepa(env: Env, config: GEPAConfig) -> GEPAResult:
             await append_episode(run_dir, episode, write_lock)
 
     try:
-        # The endpoint stays config: each rollout builds and closes its own client.
+        # The endpoint stays config: the interception server builds the live client.
         ctx = ModelContext(
             client=config.client, model=config.model, sampling=config.sampling
         )

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/claude_code/harness.py

@@ -2,6 +2,8 @@
 
 import shlex
 
+from pydantic import Field
+
 from verifiers.v1.acp import ACP
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
@@ -11,31 +13,30 @@ from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
 
-CLAUDE_ACP_DIR = "/var/tmp/vf-claude-agent-acp"
+CLAUDE_ACP_DIR = "/var/tmp/vf-claude-agent-acp-{version}-{acp_version}"
 PACKAGES_DIR = f"{CLAUDE_ACP_DIR}/packages"
-ACP_VERSION = "0.63.0"
+ACP_VERSION = "0.65.0"
+CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude"
 ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp"
-ACP_COMMAND = [f"{NODE_BIN_DIR}/node", ACP_BIN]
 CLAUDE_CONFIG_ROOT = ".vf-claude"
 SKILLS_DIR = ".claude/skills"
 ACP_INSTALL = r"""
 set -e
 export PATH="/var/tmp/vf-node/bin:$PATH"
-if [ "$(cat /var/tmp/vf-claude-agent-acp/.version 2>/dev/null)" = "$VF_CLAUDE_ACP_VERSION" ] \
-    && [ -x /var/tmp/vf-claude-agent-acp/packages/node_modules/.bin/claude-agent-acp ]; then
-    exit 0
-fi
-npm install --prefix /var/tmp/vf-claude-agent-acp/packages --ignore-scripts --no-audit --no-fund \
+rm -f {ready}
+npm install --prefix {packages} --no-audit --no-fund \
     --omit=dev \
+    "@anthropic-ai/claude-code@$VF_CLAUDE_CODE_VERSION" \
     "@agentclientprotocol/claude-agent-acp@$VF_CLAUDE_ACP_VERSION" >/dev/null
-printf %s "$VF_CLAUDE_ACP_VERSION" > /var/tmp/vf-claude-agent-acp/.version
+touch {ready}
 """
 
 CLAUDE_ACP = ACP()
 
 
 class ClaudeCodeHarnessConfig(HarnessConfig):
-    pass
+    version: str = Field(default="2.1.223", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Claude Code release to install, pinned for reproducibility."""
 
 
 class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
@@ -47,15 +48,26 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
     async def setup(self, runtime: Runtime) -> None:
         await self.install_skills(runtime, SKILLS_DIR)
         await ensure_node(runtime)
+        versions = {"version": self.config.version, "acp_version": ACP_VERSION}
+        directory = CLAUDE_ACP_DIR.format(**versions)
+        packages = PACKAGES_DIR.format(**versions)
+        claude_bin = CLAUDE_BIN.format(**versions)
+        acp_bin = ACP_BIN.format(**versions)
+        ready = f"{directory}/.ready"
+        script = ACP_INSTALL.replace("{packages}", packages).replace("{ready}", ready)
+        ensure = shlex.quote(
+            f"[ -f {ready} ] && [ -x {claude_bin} ] && [ -x {acp_bin} ] || ({script})"
+        )
         acp_guarded = (
-            f"mkdir -p {CLAUDE_ACP_DIR} && "
-            f'"$(command -v flock || command -v lockf)" {CLAUDE_ACP_DIR}/install.lock '
-            f"sh -c {shlex.quote(ACP_INSTALL)}"
+            f"mkdir -p {directory} && "
+            f'"$(command -v flock || command -v lockf)" {directory}/install.lock '
+            f"sh -c {ensure}"
         )
         acp_result = await runtime.run(
             ["sh", "-c", acp_guarded],
             {
                 **self.config.resolved_env,
+                "VF_CLAUDE_CODE_VERSION": self.config.version,
                 "VF_CLAUDE_ACP_VERSION": ACP_VERSION,
             },
         )
@@ -80,6 +92,7 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
             )
         system_prompt, prompt = self.resolve_prompt(data)
         config_dir = self.config_dir(trace)
+        versions = {"version": self.config.version, "acp_version": ACP_VERSION}
         options: dict[str, object] = {
             "strictMcpConfig": True,
             "disallowedTools": self.config.disabled_tools or [],
@@ -92,6 +105,7 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
             "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
             "ANTHROPIC_API_KEY": secret,
             "ANTHROPIC_MODEL": ctx.model,
+            "CLAUDE_CODE_EXECUTABLE": CLAUDE_BIN.format(**versions),
             "CLAUDE_CONFIG_DIR": config_dir,
             "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
             "DISABLE_AUTOUPDATER": "1",
@@ -107,7 +121,7 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
             mcp_urls,
             data,
             env=env,
-            command=ACP_COMMAND,
+            command=[f"{NODE_BIN_DIR}/node", ACP_BIN.format(**versions)],
             prompt=prompt or "",
             session_meta=session_meta,
         )
@@ -124,6 +138,7 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(data)
         config_dir = self.config_dir(trace)
+        versions = {"version": self.config.version, "acp_version": ACP_VERSION}
 
         options: dict[str, object] = {
             "strictMcpConfig": True,
@@ -138,6 +153,7 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
             "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
             "ANTHROPIC_API_KEY": secret,
             "ANTHROPIC_MODEL": ctx.model,
+            "CLAUDE_CODE_EXECUTABLE": CLAUDE_BIN.format(**versions),
             "CLAUDE_CONFIG_DIR": config_dir,
             "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
             "DISABLE_AUTOUPDATER": "1",
@@ -146,7 +162,7 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
         return await CLAUDE_ACP.run(
             runtime,
             env,
-            ACP_COMMAND,
+            [f"{NODE_BIN_DIR}/node", ACP_BIN.format(**versions)],
             prompt or "",
             mcp_urls=mcp_urls,
             session_path=f"{config_dir}/acp-session",

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/codex/harness.py

@@ -7,6 +7,8 @@ import re
 import shlex
 from collections import Counter
 
+from pydantic import Field
+
 from verifiers.v1.acp import ACP
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
@@ -21,33 +23,29 @@ logger = logging.getLogger(__name__)
 PROVIDER = "intercept"
 KEY_VAR = "CODEX_INTERCEPT_KEY"
 
-CODEX_DIR = "/var/tmp/vf-codex"
+CODEX_DIR = "/var/tmp/vf-codex-{version}-{acp_version}"
 PACKAGES_DIR = f"{CODEX_DIR}/acp"
-CODEX_VERSION = "0.145.0"
-ACP_VERSION = "1.1.7"
+ACP_VERSION = "1.1.10"
+CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex"
 ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp"
-ACP_COMMAND = [f"{NODE_BIN_DIR}/node", ACP_BIN]
 SKILLS_DIR = ".agents/skills"
 INSTALL = r"""
 set -e
 export PATH="/var/tmp/vf-node/bin:$PATH"
-versions="$VF_CODEX_VERSION:$VF_CODEX_ACP_VERSION"
-if [ "$(cat /var/tmp/vf-codex/.versions 2>/dev/null)" = "$versions" ] \
-    && [ -x /var/tmp/vf-codex/acp/node_modules/.bin/codex ] \
-    && [ -x /var/tmp/vf-codex/acp/node_modules/.bin/codex-acp ]; then
-    exit 0
-fi
-npm install --prefix /var/tmp/vf-codex/acp --ignore-scripts --no-audit --no-fund \
+rm -f {ready}
+npm install --prefix {packages} --ignore-scripts --no-audit --no-fund \
     --omit=dev \
     "@agentclientprotocol/codex-acp@$VF_CODEX_ACP_VERSION" \
     "@openai/codex@$VF_CODEX_VERSION" >/dev/null
-printf %s "$versions" > /var/tmp/vf-codex/.versions
+touch {ready}
 """
 
 CODEX_ACP = ACP()
 
 
 class CodexHarnessConfig(HarnessConfig):
+    version: str = Field(default="0.146.1", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Codex release to install, pinned for reproducibility."""
     multi_agent: bool = False
     """Enable Codex's native multi-agent v2 tools."""
 
@@ -63,19 +61,29 @@ class CodexHarness(Harness[CodexHarnessConfig]):
         await ensure_node(runtime)
         logger.info(
             "codex: ensuring Codex %s and codex-acp %s are installed",
-            CODEX_VERSION,
+            self.config.version,
             ACP_VERSION,
         )
+        versions = {"version": self.config.version, "acp_version": ACP_VERSION}
+        directory = CODEX_DIR.format(**versions)
+        packages = PACKAGES_DIR.format(**versions)
+        codex_bin = CODEX_BIN.format(**versions)
+        acp_bin = ACP_BIN.format(**versions)
+        ready = f"{directory}/.ready"
+        script = INSTALL.replace("{packages}", packages).replace("{ready}", ready)
+        ensure = shlex.quote(
+            f"[ -f {ready} ] && [ -x {codex_bin} ] && [ -x {acp_bin} ] || ({script})"
+        )
         guarded = (
-            f"mkdir -p {CODEX_DIR} && "
-            f'"$(command -v flock || command -v lockf)" {CODEX_DIR}/install.lock '
-            f"sh -c {shlex.quote(INSTALL)}"
+            f"mkdir -p {directory} && "
+            f'"$(command -v flock || command -v lockf)" {directory}/install.lock '
+            f"sh -c {ensure}"
         )
         install = await runtime.run(
             ["sh", "-c", guarded],
             {
                 **self.config.resolved_env,
-                "VF_CODEX_VERSION": CODEX_VERSION,
+                "VF_CODEX_VERSION": self.config.version,
                 "VF_CODEX_ACP_VERSION": ACP_VERSION,
             },
         )
@@ -112,7 +120,10 @@ class CodexHarness(Harness[CodexHarnessConfig]):
             {},
             data,
             env=env,
-            command=ACP_COMMAND,
+            command=[
+                f"{NODE_BIN_DIR}/node",
+                ACP_BIN.format(version=self.config.version, acp_version=ACP_VERSION),
+            ],
             prompt=prompt,
             system_prompt=system_prompt,
         )
@@ -139,7 +150,10 @@ class CodexHarness(Harness[CodexHarnessConfig]):
         return await CODEX_ACP.run(
             runtime,
             env,
-            ACP_COMMAND,
+            [
+                f"{NODE_BIN_DIR}/node",
+                ACP_BIN.format(version=self.config.version, acp_version=ACP_VERSION),
+            ],
             prompt,
             system_prompt=system_prompt,
             session_path=f"{self.trace_home(trace)}/acp-session",

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -4,6 +4,8 @@ import json
 import logging
 import shlex
 
+from pydantic import Field
+
 from verifiers.v1.acp import ACP
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
@@ -43,7 +45,7 @@ KIMI_ACP = ACP()
 
 
 class KimiCodeHarnessConfig(HarnessConfig):
-    version: str = "0.29.0"
+    version: str = Field(default="0.34.0", pattern=r"^[A-Za-z0-9._+-]+$")
     """Kimi Code release to install, pinned for reproducibility."""
 
 

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/mini_swe_agent/harness.py

@@ -1,5 +1,7 @@
 from pathlib import Path
 
+from pydantic import Field
+
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.harness import Harness
@@ -11,7 +13,7 @@ PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
 
 
 class MiniSWEAgentHarnessConfig(HarnessConfig):
-    version: str = "2.4.5"
+    version: str = Field(default="2.4.6", pattern=r"^[A-Za-z0-9._+-]+$")
     """mini-swe-agent release to install, pinned for reproducibility."""
 
 

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/openclaw/harness.py

@@ -117,10 +117,8 @@ class OpenClawHarness(Harness[OpenClawHarnessConfig]):
         data: TaskData,
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(data)
-        reasoning = ctx.sampling.reasoning_effort not in (
-            None,
-            "none",
-        ) or ctx.model.rsplit("/", 1)[-1].startswith(("gpt-5", "o1", "o3", "o4"))
+        # Opt-in via sampling: OpenClaw redacts persisted encrypted reasoning, so resumed sessions 400 replaying it.
+        reasoning = ctx.sampling.reasoning_effort not in (None, "none")
         directory = OPENCLAW_DIR.format(version=self.config.version)
         state_dir = f".vf-openclaw/{trace.id}"
         config_path = f"{state_dir}/openclaw.json"

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/pi/harness.py

@@ -4,6 +4,8 @@ import json
 import logging
 import shlex
 
+from pydantic import Field
+
 from verifiers.v1.acp import ACP
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
@@ -23,8 +25,8 @@ PI_DIR = "/var/tmp/vf-pi"
 PACKAGES_DIR = f"{PI_DIR}/mcp"
 PI_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi"
 SKILLS_DIR = ".agents/skills"
-MCP_VERSION = "2.11.0"
-ACP_VERSION = "0.0.31"
+MCP_VERSION = "2.20.1"
+ACP_VERSION = "0.0.33"
 MCP_ADAPTER = f"{PACKAGES_DIR}/node_modules/pi-mcp-adapter/index.ts"
 ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi-acp"
 ACP_COMMAND = [
@@ -86,7 +88,7 @@ PI_ACP = ACP()
 
 
 class PiHarnessConfig(HarnessConfig):
-    version: str = "0.80.10"
+    version: str = Field(default="0.84.0", pattern=r"^[A-Za-z0-9._+-]+$")
     """Pi release to install, pinned for reproducibility."""
 
 
@@ -223,4 +225,6 @@ class PiHarness(Harness[PiHarnessConfig]):
             ACP_COMMAND,
             prompt,
             session_path=f"{agent_dir}/acp-session",
+            # Pi can end after its final tool completes without a text message.
+            allow_empty_tool_reply=True,
         )

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/pool/harness.py

@@ -31,7 +31,7 @@ POOL_ACP = ACP()
 
 
 class PoolHarnessConfig(HarnessConfig):
-    version: str = Field(default="1.0.11", pattern=r"^[A-Za-z0-9._+-]+$")
+    version: str = Field(default="1.0.15", pattern=r"^[A-Za-z0-9._+-]+$")
     """Pool release to install, pinned for reproducibility."""
 
 

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/rlm/harness.py

@@ -30,7 +30,7 @@ RLM_ACP = ACP()
 
 
 class RLMHarnessConfig(HarnessConfig):
-    version: str = "main"
+    version: str = Field(default="main", min_length=1)
     """Git ref (branch, tag, or commit) of rlm-harness to install."""
     max_depth: int = 0
     """Recursion depth rlm may spawn sub-harnesses to (RLM_MAX_DEPTH)."""

{verifiers-0.2.2.dev96 → verifiers-0.2.2.dev112}/verifiers/v1/harnesses/terminus_2/harness.py

@@ -1,6 +1,8 @@
 import logging
 from pathlib import Path
 
+from pydantic import Field
+
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.harness import Harness
@@ -13,7 +15,7 @@ logger = logging.getLogger(__name__)
 
 
 class Terminus2HarnessConfig(HarnessConfig):
-    version: str = "0.20.0"
+    version: str = Field(default="0.20.0", pattern=r"^[A-Za-z0-9._+-]+$")
     """Harbor release to install, pinned for reproducibility."""