verifiers 0.3.2.dev107__tar.gz → 0.3.2.dev109__tar.gz

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev107
+Version: 0.3.2.dev109
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -36,7 +36,7 @@ Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
 Requires-Dist: prime-runs>=0.1.3
-Requires-Dist: prime-sandboxes>=0.2.39
+Requires-Dist: prime-sandboxes>=0.3.0
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "math-verify>=0.8.0",
     "openai>=2.54.0,<3.0.0",
     "prime-tunnel>=0.1.8",
-    "prime-sandboxes>=0.2.39",
+    "prime-sandboxes>=0.3.0",
     "prime-runs>=0.1.3",
     "pydantic>=2.12.3",
     "rich>=11.0.0",

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/tests/v1/test_e2e.py

@@ -356,10 +356,7 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     (trace,) = await run_v1(
         "echo-acp-resume-v1",
         harness=harness,
-        runtime={
-            "type": harness_runtime,
-            **({"vm": True} if harness_runtime == "prime" else {}),
-        },
+        runtime={"type": harness_runtime},
         output_dir=tmp_path,
         max_turns=8,
         max_tokens=8192,

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/agent.py

@@ -540,6 +540,13 @@ class Agent:
     ) -> dict:
         """Resolve one run's runtime config, pairing checks, timeouts,
         interception — shared by `run` and `interaction`."""
+        harness = self.harness
+        skills = [*task.data.skills, *harness.config.skills]
+        if skills:
+            # Skill installations can hold run-specific state, such as RLM's package environment.
+            harness = type(harness)(
+                harness.config.model_copy(update={"skills": skills})
+            )
         if runtime is not None:
             _check_borrowed_placement(task, runtime, self.runtime_config)
             runtime_config = runtime.config
@@ -550,7 +557,7 @@ class Agent:
             )
             run_is_local = runtime_is_local(runtime_config)
         validate_pairing(
-            self.harness,
+            harness,
             type(task),
             runtime_config,
             tools=[*task.toolsets(task.config), *shared_tools.values()],
@@ -558,7 +565,7 @@ class Agent:
         timeouts = resolve_rollout_timeouts(self.timeout, task)
         return {
             "agent_config": self.config,
-            "harness": self.harness,
+            "harness": harness,
             "ctx": self.ctx,
             "runtime_config": runtime_config,
             "timeouts": replace(

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/configs/agent.py

@@ -14,7 +14,7 @@ class TimeoutConfig(BaseConfig):
     """Timeout (in seconds) for different phases of an agent's run."""
 
     setup: float | None = None  # one shared budget: task setup + provisioning
-    """Timeout (in seconds) for the task + harness setup hooks."""
+    """Timeout (in seconds) for task/harness setup through session preparation."""
     rollout: float | None = None
     """Timeout (in seconds) for the agent's solve attempt. Unset: the task's own
     timeout, else 4 hours. `0` disables the timeout."""

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/configs/harness.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import os
 from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Literal
 
 from pydantic import ConfigDict, Field, FiniteFloat
 from pydantic_config import BaseConfig
@@ -15,6 +15,14 @@ PinnedVersion = Annotated[str, Field(pattern=r"^[A-Za-z0-9._+-]+$")]
 """A release/tag a harness pins its program install to."""
 
 
+SkillSource = Path | Annotated[dict[Literal["runtime"], str], Field(min_length=1)]
+
+
+def skill_destination(skill: SkillSource, dest: str) -> str:
+    """Runtime roots merge into `dest`; host folders keep their resolved name."""
+    return dest if isinstance(skill, dict) else f"{dest}/{skill.resolve().name}"
+
+
 class HarnessConfig(BaseConfig):
     id: ID = "bash"
     """Installed harness package, set through the seat's
@@ -44,10 +52,10 @@ class HarnessConfig(BaseConfig):
     tool_timeout: FiniteFloat = Field(600.0, gt=0)
     """Seconds a single MCP tool call may take; raise it for tools that boot a VM."""
     disabled_tools: list[str] | None = None
-    skills: list[Path] = Field(default_factory=list)
-    """Skill folders to upload into the program's skill discovery directory — each
-    lands at `<skills dir>/<folder name>`. Only harnesses whose program discovers
-    skills natively (`SUPPORTS_SKILLS`) accept them."""
+    skills: list[SkillSource] = Field(default_factory=list)
+    """Host skill folders or `{runtime: path}` roots of skills inside the runtime.
+    Sources are installed in order; later files override matching earlier files.
+    Only harnesses with native skill support (`SUPPORTS_SKILLS`) accept them."""
 
     @property
     def name(self) -> str:

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harness.py

@@ -8,7 +8,7 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
 
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.configs.harness import HarnessConfig, skill_destination
 from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
@@ -45,8 +45,8 @@ class Harness(ABC, Generic[ConfigT]):
     where model-directed execution changes the rules: the subprocess-on-host
     warning, the judge env's sandbox requirement."""
     SUPPORTS_SKILLS: ClassVar[bool] = False
-    """Whether the program discovers SKILL.md skills — its `setup` calls
-    `install_skills` with the program's fixed discovery location; configuring
+    """Whether the program discovers SKILL.md skills — it calls
+    `install_skills` with a run-scoped discovery location; configuring
     `skills` on a harness without support is rejected up front."""
     NEEDS_CONTAINER: ClassVar[bool] = True
     """Whether the program must run in a container runtime: True for every harness
@@ -97,35 +97,64 @@ class Harness(ABC, Generic[ConfigT]):
         """Provision this harness in `runtime` before its execution timeout starts."""
 
     async def install_skills(self, runtime: Runtime, dest: str) -> None:
-        """Upload each `config.skills` folder into `runtime` at `dest/<folder name>` —
-        the program's fixed skill discovery location, which a supporting harness's
-        `setup` passes."""
-        uploads = []
-        executables = []
+        """Install `config.skills` in the program's discovery directory.
+        Runtime roots are copied in place; host skill folders are uploaded."""
         for skill in self.config.skills:
+            target_dir = skill_destination(skill, dest)
+            if isinstance(skill, dict):
+                result = await runtime.run(
+                    [
+                        "bash",
+                        "-e",
+                        "-o",
+                        "pipefail",
+                        "-c",
+                        r"""
+[ -d "$1" ]
+source=$(CDPATH= cd -- "$1/." && pwd -P)
+mkdir -p -- "$2"
+target=$(CDPATH= cd -- "$2/." && pwd -P)
+[ "$source" != "$target" ] || exit 0
+# A source such as "." must not copy the destination back into itself.
+exclude=$(printf '%s' "./${target#"${source%/}/"}" | sed 's/[][\\*?]/\\&/g')
+tar -C "$source" --exclude="$exclude" -cf - . | tar -xpf - -C "$target"
+""",
+                        "vf-skills",
+                        skill["runtime"],
+                        target_dir,
+                    ],
+                    {},
+                )
+                if result.exit_code:
+                    raise RuntimeError(
+                        f"installing runtime skills from {skill['runtime']!r} failed: {result.stderr}"
+                    )
+                continue
             # Resolve so `.`/`..` entries get their real folder name (and can't
             # place files outside `dest`).
             skill = skill.resolve()
             if not skill.is_dir():
                 raise ValueError(f"skill {str(skill)!r} is not a folder")
+            uploads = []
+            executables = []
             for file in skill.rglob("*"):
                 if not file.is_file():
                     continue
-                target = f"{dest}/{skill.name}/{file.relative_to(skill).as_posix()}"
+                target = f"{target_dir}/{file.relative_to(skill).as_posix()}"
                 uploads.append((target, file.read_bytes()))
                 if os.access(file, os.X_OK):
                     executables.append(target)
-        # Settle every upload before returning or surfacing an error.
-        results = await asyncio.gather(
-            *(runtime.write(target, data) for target, data in uploads),
-            return_exceptions=True,
-        )
-        for result in results:
-            if isinstance(result, BaseException):
-                raise result
-        if executables:
-            # `write` moves bytes, not modes; restore the execute bits scripts need.
-            await runtime.run(["chmod", "+x", *executables], {})
+            # Finish this source before the next one overwrites matching files.
+            results = await asyncio.gather(
+                *(runtime.write(target, data) for target, data in uploads),
+                return_exceptions=True,
+            )
+            for result in results:
+                if isinstance(result, BaseException):
+                    raise result
+            if executables:
+                # `write` moves bytes, not modes; restore the execute bits scripts need.
+                await runtime.run(["chmod", "+x", *executables], {})
 
     async def _check_result(
         self, trace: Trace, runtime: Runtime, result: ProgramResult

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harnesses/claude_code/harness.py

@@ -15,7 +15,6 @@ ACP_VERSION = "0.67.0"
 CLAUDE_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude"
 ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/claude-agent-acp"
 CLAUDE_CONFIG_ROOT = ".vf-claude"
-SKILLS_DIR = ".claude/skills"
 ACP_INSTALL = r"""
 set -e
 export PATH="/var/tmp/vf-node/bin:$PATH"
@@ -39,7 +38,6 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]):
     SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await self.install_skills(runtime, SKILLS_DIR)
         await ensure_node(runtime)
         versions = {"version": self.config.version, "acp_version": ACP_VERSION}
         directory = CLAUDE_ACP_DIR.format(**versions)
@@ -74,6 +72,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]):
     ) -> ACPConfig:
         system_prompt, prompt = self.resolve_prompt(data)
         config_dir = self.config_dir(trace)
+        await self.install_skills(runtime, f"{config_dir}/skills")
         versions = {"version": self.config.version, "acp_version": ACP_VERSION}
         session_meta = {
             "claudeCode": {

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harnesses/codex/harness.py

@@ -22,7 +22,6 @@ PACKAGES_DIR = f"{CODEX_DIR}/acp"
 ACP_VERSION = "1.2.0"
 CODEX_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex"
 ACP_BIN = f"{PACKAGES_DIR}/node_modules/.bin/codex-acp"
-SKILLS_DIR = ".agents/skills"
 INSTALL = r"""
 set -e
 export PATH="/var/tmp/vf-node/bin:$PATH"
@@ -58,7 +57,6 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]):
             raise RuntimeError(f"Codex {failure['category']}: {failure['safeMessage']}")
 
     async def setup(self, runtime: Runtime) -> None:
-        await self.install_skills(runtime, SKILLS_DIR)
         await ensure_node(runtime)
         logger.info(
             "codex: ensuring Codex %s and codex-acp %s are installed",
@@ -137,6 +135,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]):
         mcp_urls: dict[str, str],
     ) -> dict[str, str]:
         home = self.trace_home(trace)
+        await self.install_skills(runtime, f"{home}/skills")
         mcp_config = "features={mcp_2026_07_28=true}\n" + (
             "mcp_servers={"
             + ",".join(

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -8,7 +8,7 @@ import tomli_w
 from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
-from verifiers.v1.harnesses.utils.install import ensure_installed
+from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
 from verifiers.v1.runtimes import Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
@@ -18,7 +18,6 @@ logger = logging.getLogger(__name__)
 BINARY = "/tmp/vf-kimi-code/bin/kimi"
 KIMI_HOME = ".vf-kimi-code"
 ACP_COMMAND = [BINARY, "acp"]
-SKILLS_DIR = f"{KIMI_HOME}/skills"
 
 INSTALL = r"""
 set -e
@@ -52,7 +51,6 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]):
     SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await self.install_skills(runtime, SKILLS_DIR)
         logger.info(
             "kimi-code: ensuring Kimi Code %s is installed", self.config.version
         )
@@ -77,6 +75,8 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]):
         data: TaskData,
     ) -> ACPConfig:
         kimi_home = f"{KIMI_HOME}/{trace.id}"
+        skills_dir = f"{kimi_home}/skills"
+        await self.install_skills(runtime, skills_dir)
         provider_type = {
             "chat_completions": "openai",
             "responses": "openai_responses",
@@ -88,7 +88,7 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]):
             else endpoint
         )
         config = {
-            "extra_skill_dirs": [SKILLS_DIR] if self.config.skills else [],
+            "extra_skill_dirs": [skills_dir] if self.config.skills else [],
             **(
                 {
                     "permission": {
@@ -125,3 +125,6 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]):
             prompt=prompt,
             system_prompt=system_prompt,
         )
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        await remove_dir(runtime, f"{KIMI_HOME}/{trace.id}", "Kimi Code state")

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harnesses/openclaw/harness.py

@@ -1,9 +1,7 @@
 """Run OpenClaw's Gateway-backed ACP agent against interception."""
 
-import asyncio
 import json
 import logging
-import secrets
 
 from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.clients import ModelContext
@@ -130,22 +128,6 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
     SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
-        if not hasattr(self, "_staged_skills_dir"):
-            self._staged_skills_dir = (
-                f".vf-openclaw/staged-skills-{secrets.token_hex(8)}"
-            )
-            self._skills_setup_lock = asyncio.Lock()
-        if self.config.skills:
-            async with self._skills_setup_lock:
-                # A complete tree is immutable, so concurrent setups can safely reuse it.
-                ready_path = f"{self._staged_skills_dir}/.ready"
-                ready = await runtime.run(["test", "-f", ready_path], {})
-                if ready.exit_code != 0:
-                    await remove_dir(
-                        runtime, self._staged_skills_dir, "OpenClaw skills"
-                    )
-                    await self.install_skills(runtime, self._staged_skills_dir)
-                    await runtime.write(ready_path, b"")
         directory = OPENCLAW_DIR.format(version=self.config.version)
         binary = OPENCLAW_BIN.format(version=self.config.version)
         logger.info("openclaw: ensuring OpenClaw %s is installed", self.config.version)
@@ -233,14 +215,7 @@ class OpenClawHarness(ACPHarness[OpenClawHarnessConfig]):
             # OpenClaw treats an empty allowlist as all; a no-match key disables the catalog.
             config.setdefault("skills", {})["allowBundled"] = ["__none__"]
         await runtime.write(config_path, json.dumps(config).encode())
-        if self.config.skills:
-            copied = await runtime.run(
-                ["cp", "-R", self._staged_skills_dir, skills_dir], {}
-            )
-            if copied.exit_code != 0:
-                raise RuntimeError(
-                    f"failed to isolate OpenClaw skills: {copied.stderr.strip()[-500:]}"
-                )
+        await self.install_skills(runtime, skills_dir)
 
         env = {
             **self.config.resolved_env,

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harnesses/pi/harness.py

@@ -7,9 +7,9 @@ from typing import Literal
 
 from verifiers.v1.acp import ACPConfig, ACPHarness
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
+from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion, skill_destination
 from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
-from verifiers.v1.harnesses.utils.install import ensure_installed
+from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
 from verifiers.v1.runtimes import Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
@@ -21,7 +21,6 @@ KEY_VAR = "PI_INTERCEPT_KEY"
 PI_DIR = "/var/tmp/vf-pi"
 PACKAGES_DIR = f"{PI_DIR}/mcp"
 PI_BIN = f"{PACKAGES_DIR}/node_modules/.bin/pi"
-SKILLS_DIR = ".agents/skills"
 MCP_VERSION = "2.28.0"
 ACP_VERSION = "0.0.33"
 MCP_ADAPTER = f"{PACKAGES_DIR}/node_modules/pi-mcp-adapter/index.ts"
@@ -61,7 +60,6 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
     SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
-        await self.install_skills(runtime, SKILLS_DIR)
         await ensure_node(runtime)
         logger.info(
             "pi: ensuring Pi %s and pi-acp %s are installed",
@@ -93,6 +91,8 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
     ) -> ACPConfig:
         system_prompt, prompt = self.resolve_prompt(data)
         agent_dir = f".vf-pi-agent-{trace.id}"
+        skills_dir = f"{agent_dir}/skills"
+        await self.install_skills(runtime, skills_dir)
         reasoning = ctx.sampling.reasoning_effort not in (
             None,
             "none",
@@ -159,8 +159,7 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
         skill_args = [
             arg
             for skill in self.config.skills
-            # Resolve like `install_skills` so the path matches what it wrote.
-            for arg in ("--skill", f"{SKILLS_DIR}/{skill.resolve().name}")
+            for arg in ("--skill", skill_destination(skill, skills_dir))
         ]
         pi_args = [
             PI_BIN,
@@ -190,3 +189,6 @@ class PiHarness(ACPHarness[PiHarnessConfig]):
             # Pi's extension owns the task-scoped MCP configuration.
             mcp_urls={},
         )
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        await remove_dir(runtime, f".vf-pi-agent-{trace.id}", "Pi state")

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harnesses/prime_agent/harness.py

@@ -7,7 +7,7 @@ from typing import Literal
 
 from verifiers.v1.acp import ACPConfig, ACPHarness, ACPTurn
 from verifiers.v1.clients import ModelContext
-from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.configs.harness import HarnessConfig, skill_destination
 from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
 from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
 from verifiers.v1.runtimes import Runtime
@@ -25,7 +25,6 @@ PRIME_AGENT_COMMIT: Literal["81ae3cb34d27d38ee37f9e205a1e73694993b344"] = (
 PRIME_AGENT_VERSION = "0.9.1"
 PRIME_AGENT_DIR = "/var/tmp/vf-prime-agent"
 STATE_ROOT = "/tmp/vf-prime-agent-runs"
-SKILLS_DIR = ".agents/skills"
 PROVIDER = "intercept"
 LIFECYCLE_META_NAMESPACE = "ai.primeintellect.prime-agent"
 KEY_VAR = "PRIME_AGENT_INTERCEPT_KEY"
@@ -155,7 +154,6 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
         statuses.append(status)
 
     async def setup(self, runtime: Runtime) -> None:
-        await self.install_skills(runtime, SKILLS_DIR)
         await ensure_node(runtime)
         logger.info("prime-agent: ensuring commit %s is installed", self.config.commit)
         await ensure_installed(
@@ -191,6 +189,7 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
 
         root = self._root(trace)
         agent_dir = f"{root}/agent"
+        skills_dir = f"{agent_dir}/skills"
         created = await runtime.run(
             [
                 "mkdir",
@@ -207,6 +206,7 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
             raise RuntimeError(
                 f"prime-agent state directory failed: {created.stderr.strip()[-500:]}"
             )
+        await self.install_skills(runtime, skills_dir)
         reasoning = ctx.sampling.reasoning_effort not in (
             None,
             "none",
@@ -251,7 +251,7 @@ class PrimeAgentHarness(ACPHarness[PrimeAgentHarnessConfig]):
         if self.config.autonomous:
             args.append("--autonomous")
         for skill in self.config.skills:
-            args += ["--skill", f"{SKILLS_DIR}/{skill.resolve().name}"]
+            args += ["--skill", skill_destination(skill, skills_dir)]
         if system_prompt:
             args += ["--append-system-prompt", system_prompt]
 

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/harnesses/rlm/harness.py

@@ -2,6 +2,7 @@
 
 import hashlib
 import logging
+import secrets
 import shlex
 from typing import Any, Literal
 
@@ -18,7 +19,7 @@ from pydantic_config import BaseConfig
 from verifiers.v1.acp import ACPConfig, ACPHarness, ACPTurn, JsonObject
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
-from verifiers.v1.harnesses.utils.install import ensure_installed
+from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
 from verifiers.v1.runtimes import Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
@@ -140,11 +141,15 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_SKILLS = True
+    _skills_install_dir: str | None = None
 
     async def setup(self, runtime: Runtime) -> None:
-        # Before the installer: install.sh packages the skills it finds.
-        await self.install_skills(runtime, SKILLS_DIR)
+        if self.config.skills:
+            # Editable skill packages and uv's tool environment must belong to this run.
+            self._skills_install_dir = f"{RLM_CACHE_DIR}-skills-{secrets.token_hex(16)}"
         directory = self._install_dir()
+        skills_dir = f"{directory}/skills" if self.config.skills else SKILLS_DIR
+        await self.install_skills(runtime, skills_dir)
         binary = f"{directory}/bin/rlm"
         checkout = f"{directory}/checkout"
         ready = f"{directory}/.ready"
@@ -155,7 +160,9 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]):
             "{ apt-get update -qq && apt-get install -y -qq git; } && "
             f"rm -rf {checkout} && git clone https://{RLM_REPO} {checkout} && "
             f"git -C {checkout} checkout {shlex.quote(self.config.version)} && "
+            f"sed -i 's|/task/rlm-skills|{skills_dir}|g' {checkout}/install.sh && "
             f"UV_INSTALL_DIR={directory}/bin UV_TOOL_BIN_DIR={directory}/bin "
+            f"UV_TOOL_DIR={directory}/tools "
             f"RLM_CHECKOUT_PATH={checkout} bash {checkout}/install.sh && "
             f"touch {ready})"
         )
@@ -279,6 +286,10 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]):
 
     async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
         await runtime.run(["rm", "-rf", f"{RLM_STATE_DIR}/{trace.id}"], {})
+        if self._skills_install_dir is not None:
+            await remove_dir(
+                runtime, self._skills_install_dir, "RLM skill installation"
+            )
 
     @staticmethod
     def _home(trace: Trace) -> str:
@@ -286,4 +297,4 @@ class RLMHarness(ACPHarness[RLMHarnessConfig]):
 
     def _install_dir(self) -> str:
         cache_key = hashlib.sha256(self.config.version.encode()).hexdigest()
-        return f"{RLM_CACHE_DIR}-{cache_key}"
+        return self._skills_install_dir or f"{RLM_CACHE_DIR}-{cache_key}"

{verifiers-0.3.2.dev107 → verifiers-0.3.2.dev109}/verifiers/v1/rollout.py

@@ -43,7 +43,7 @@ class RolloutTimeouts:
     """Per-stage rollout timeouts, each bounding one rollout stage."""
 
     setup: float | None = None
-    """Timeout (in seconds) for the task + harness setup hooks."""
+    """Timeout (in seconds) for task/harness setup through session preparation."""
     agent: float | None = None
     """Timeout (in seconds) for the agent's solve attempt."""
     finalize: float | None = None
@@ -221,7 +221,7 @@ class Rollout:
             self.trace.timing.boot.end = now
             self.trace.timing.setup.start = now
             self.trace.notify()
-            # Task setup and harness provisioning share one setup-stage deadline.
+            # Setup hooks and harness session preparation share one deadline.
             setup_deadline = (
                 None
                 if self._timeouts.setup is None
@@ -271,7 +271,10 @@ class Rollout:
             # Setup and service provisioning are complete. Apply the runtime's
             # execution policy while preserving the framework routes the agent uses.
             await runtime.prepare_execution([self._endpoint, *self._urls.values()])
-            async with boundary(HarnessError, "opening harness session"):
+            async with (
+                boundary(HarnessError, "opening harness session"),
+                asyncio.timeout_at(setup_deadline),
+            ):
                 harness_data = self.trace.task.data
                 if (
                     self._session.request_interceptors