verifiers 0.2.2.dev17__tar.gz → 0.2.2.dev18__tar.gz

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev17
+Version: 0.2.2.dev18
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/tests/v1/test_e2e.py

@@ -15,26 +15,28 @@ def _pair(a: str, b: str, id: str, *extra_marks):
     return pytest.param(a, b, marks=[*marks, *extra_marks], id=id)
 
 
-# harness x harness runtime: every harness once, both local runtimes hit, one remote row
-# per provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
+# harness x harness runtime: every harness once, both local runtimes hit (subprocess
+# only carries the in-house loops — the rest NEEDS_CONTAINER), one remote row per
+# provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
 # task) — test_agentic covers them.
 CHAT_PLACEMENTS = [
     _pair("null", "subprocess", "null-harness-in-subprocess"),
     _pair("bash", "docker", "bash-harness-in-docker"),
-    _pair("rlm", "subprocess", "rlm-harness-in-subprocess"),
+    _pair("rlm", "docker", "rlm-harness-in-docker"),
     _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]
 
 # harness x harness runtime for the shell task: every coding agent once (null is a chat
-# loop with no shell), both local runtimes hit, one remote row per provider.
+# loop with no shell), both local runtimes hit (subprocess only carries bash), one
+# remote row per provider.
 AGENTIC_PLACEMENTS = [
     _pair("bash", "subprocess", "bash-harness-in-subprocess"),
     _pair("rlm", "docker", "rlm-harness-in-docker"),
-    _pair("kimi-code", "subprocess", "kimi-code-harness-in-subprocess"),
+    _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("codex", "docker", "codex-harness-in-docker"),
-    _pair("claude-code", "subprocess", "claude-code-harness-in-subprocess"),
+    _pair("claude-code", "docker", "claude-code-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harness.py

@@ -4,6 +4,7 @@ import logging
 import os
 from abc import ABC, abstractmethod
 from collections.abc import Mapping
+from pathlib import Path
 from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
 
 from pydantic import Field
@@ -41,6 +42,10 @@ class HarnessConfig(BaseConfig):
     forward_env: list[str] = Field(default_factory=list)
     """Host variables to forward without writing secrets into config; explicit `env` wins."""
     disabled_tools: list[str] | None = None
+    skills: list[Path] = Field(default_factory=list)
+    """Skill folders to upload into the program's skill discovery directory — each
+    lands at `<skills dir>/<folder name>`. Only harnesses whose program discovers
+    skills natively (`SUPPORTS_SKILLS`) accept them."""
 
     @property
     def name(self) -> str:
@@ -66,6 +71,15 @@ class Harness(ABC, Generic[ConfigT]):
     every real harness; the tool-less chat loops (`null`) override to False. Read
     where model-directed execution changes the rules: the subprocess-on-host
     warning, the judge env's sandbox requirement."""
+    SUPPORTS_SKILLS: ClassVar[bool] = False
+    """Whether the program discovers SKILL.md skills — its `setup` calls
+    `install_skills` with the program's fixed discovery location; configuring
+    `skills` on a harness without support is rejected up front."""
+    NEEDS_CONTAINER: ClassVar[bool] = True
+    """Whether the program must run in a container runtime: True for every harness
+    that installs and drives a third-party program — on the host (subprocess) it
+    leaks host state (auth, config, processes) both ways. Only the minimal
+    in-house loops (`bash`, `null`) override to False."""
 
     def __init__(self, config: ConfigT) -> None:
         self.config = config
@@ -102,6 +116,28 @@ class Harness(ABC, Generic[ConfigT]):
     async def setup(self, runtime: Runtime) -> None:
         """Provision this harness in `runtime` before its execution timeout starts."""
 
+    async def install_skills(self, runtime: Runtime, dest: str) -> None:
+        """Upload each `config.skills` folder into `runtime` at `dest/<folder name>` —
+        the program's fixed skill discovery location, which a supporting harness's
+        `setup` passes."""
+        for skill in self.config.skills:
+            # Resolve so `.`/`..` entries get their real folder name (and can't
+            # place files outside `dest`).
+            skill = skill.resolve()
+            if not skill.is_dir():
+                raise ValueError(f"skill {str(skill)!r} is not a folder")
+            executables = []
+            for file in sorted(skill.rglob("*")):
+                if not file.is_file():
+                    continue
+                target = f"{dest}/{skill.name}/{file.relative_to(skill).as_posix()}"
+                await runtime.write(target, file.read_bytes())
+                if os.access(file, os.X_OK):
+                    executables.append(target)
+            if executables:
+                # `write` moves bytes, not modes; restore the execute bits scripts need.
+                await runtime.run(["chmod", "+x", *executables], {})
+
     async def run(
         self,
         ctx: ModelContext,

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/bash/harness.py

@@ -41,6 +41,7 @@ class BashHarness(Harness[BashHarnessConfig]):
     SUPPORTS_MCP = True
     SUPPORTS_USER_SIM = True
     SUPPORTS_MESSAGE_PROMPT = True
+    NEEDS_CONTAINER = False
 
     async def setup(self, runtime: Runtime) -> None:
         await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/claude_code/harness.py

@@ -13,6 +13,7 @@ from verifiers.v1.trace import Trace
 CLAUDE_HOME = "/tmp/vf-claude-code-{version}"
 CLAUDE_BIN = f"{CLAUDE_HOME}/.local/bin/claude"
 CLAUDE_CONFIG_DIR = ".vf-claude"
+SKILLS_DIR = f"{CLAUDE_CONFIG_DIR}/skills"
 INSTALL = """
 set -e
 command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
@@ -30,8 +31,10 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
     SUPPORTS_MCP = True
     # images would require streaming inputs
     SUPPORTS_MESSAGE_PROMPT = False
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
         home = CLAUDE_HOME.format(version=self.config.version)
         binary = CLAUDE_BIN.format(version=self.config.version)
         script = INSTALL.format(version=self.config.version, home=home)
@@ -66,13 +69,13 @@ class ClaudeCodeHarness(Harness[ClaudeCodeHarnessConfig]):
             "ANTHROPIC_BASE_URL": endpoint.removesuffix("/v1"),
             "ANTHROPIC_API_KEY": secret,
             "CLAUDE_CONFIG_DIR": CLAUDE_CONFIG_DIR,
+            "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
             "DISABLE_AUTOUPDATER": "1",
             "IS_SANDBOX": "1",
         }
         argv = [
             CLAUDE_BIN.format(version=self.config.version),
             "--print",
-            "--bare",
             "--dangerously-skip-permissions",
             "--no-session-persistence",
             "--model",

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/codex/harness.py

@@ -23,6 +23,7 @@ KEY_VAR = "CODEX_INTERCEPT_KEY"
 
 CODEX_DIR = "/tmp/vf-codex"
 CODEX_BIN = f"{CODEX_DIR}/bin/codex"
+SKILLS_DIR = ".agents/skills"
 INSTALL = r"""
 set -e
 mkdir -p {dir}/bin
@@ -46,8 +47,10 @@ class CodexHarness(Harness[CodexHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = False  # TODO
     SUPPORTS_MCP = False  # TODO
     SUPPORTS_MESSAGE_PROMPT = True
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
         logger.info("codex: ensuring codex %s is installed", self.config.version)
         script = (
             INSTALL.replace("{version}", self.config.version)

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/kimi_code/harness.py

@@ -13,6 +13,7 @@ logger = logging.getLogger(__name__)
 
 BINARY = "/tmp/vf-kimi-code/bin/kimi"
 KIMI_HOME = ".vf-kimi-code"
+SKILLS_DIR = f"{KIMI_HOME}/skills"
 
 INSTALL = r"""
 set -e
@@ -39,8 +40,10 @@ class KimiCodeHarnessConfig(HarnessConfig):
 class KimiCodeHarness(Harness[KimiCodeHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = False
     SUPPORTS_MCP = True
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
         logger.info(
             "kimi-code: ensuring Kimi Code %s is installed", self.config.version
         )

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/null/harness.py

@@ -20,6 +20,7 @@ class NullHarness(Harness[NullHarnessConfig]):
     SUPPORTS_USER_SIM = True
     SUPPORTS_MESSAGE_PROMPT = True
     EXECUTES_CODE = False
+    NEEDS_CONTAINER = False
 
     async def setup(self, runtime: Runtime) -> None:
         await runtime.prepare_uv_script(PROGRAM_SOURCE, self.config.resolved_env)

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/pi/harness.py

@@ -26,6 +26,7 @@ HOME_VAR = "VF_PI_ORIGINAL_HOME"
 
 PI_DIR = "/tmp/vf-pi"
 PI_BIN = f"{PI_DIR}/pi"
+SKILLS_DIR = ".agents/skills"
 MCP_VERSION = "2.11.0"
 MCP_ADAPTER = f"{PI_DIR}/mcp/node_modules/pi-mcp-adapter/index.ts"
 
@@ -95,8 +96,12 @@ class PiHarness(Harness[PiHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_MESSAGE_PROMPT = True
+    # Pi's project skill discovery is trust-gated (a prompt print mode can't answer),
+    # so the installed skills are passed explicitly via `--skill` at launch.
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
         logger.info(
             "pi: ensuring Pi %s and pi-mcp-adapter %s are installed",
             self.config.version,
@@ -234,6 +239,12 @@ class PiHarness(Harness[PiHarnessConfig]):
             if self.config.disabled_tools
             else []
         )
+        skill_args = [
+            arg
+            for skill in self.config.skills
+            # Resolve like `install_skills` so the path matches what it wrote.
+            for arg in ("--skill", f"{SKILLS_DIR}/{skill.resolve().name}")
+        ]
         system_args = ["--append-system-prompt", system_prompt] if system_prompt else []
         argv = [
             "sh",
@@ -252,6 +263,7 @@ class PiHarness(Harness[PiHarnessConfig]):
             ctx.model,
             *mcp_args,
             *tool_args,
+            *skill_args,
             *system_args,
             *image_args,
         ]

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/pool/harness.py

@@ -13,6 +13,7 @@ from verifiers.v1.types import SystemMessage, TextContentPart, UserMessage
 
 POOL_DIR = "/tmp/vf-pool-{version}"
 SETTINGS_PATH = ".poolside/settings.local.yaml"
+SKILLS_DIR = ".poolside/skills"
 INSTALL = r"""
 set -e
 command -v curl >/dev/null || (apt-get update -qq && apt-get install -y -qq curl ca-certificates >/dev/null)
@@ -35,8 +36,10 @@ class PoolHarness(Harness[PoolHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
     SUPPORTS_MESSAGE_PROMPT = True
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
         directory = POOL_DIR.format(version=self.config.version)
         binary = f"{directory}/pool"
         script = INSTALL.replace("{version}", self.config.version).replace(

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/rlm/harness.py

@@ -24,6 +24,7 @@ RLM_REPO = "github.com/PrimeIntellect-ai/rlm.git"
 RLM_HOME = ".rlm"
 RLM_DIR = "/tmp/vf-rlm"
 RLM_BIN = f"{RLM_DIR}/bin/rlm"
+SKILLS_DIR = "/task/rlm-skills"
 
 
 class RLMHarnessConfig(HarnessConfig):
@@ -31,9 +32,9 @@ class RLMHarnessConfig(HarnessConfig):
     """Git ref (branch, tag, or commit) of rlm to install."""
     max_depth: int = 0
     """Recursion depth rlm may spawn sub-harnesses to (RLM_MAX_DEPTH)."""
-    skills: list[BuiltinSkill] = []
+    builtin_skills: list[BuiltinSkill] = []
     """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; empty enables none.
-    The tool set is fixed (ipython); only built-in skills are selectable."""
+    The tool set is fixed (ipython); the base `skills` field takes SKILL.md paths."""
     summarize_at_tokens: int | tuple[int, int] | None = None
     """Auto-compaction threshold (RLM_SUMMARIZE_AT_TOKENS): compact the context once it grows
     past this many tokens. An int is a fixed threshold; a `(lo, hi)` pair draws a per-group
@@ -63,7 +64,7 @@ class RLMHarnessConfig(HarnessConfig):
         if self.disabled_tools:
             raise ValueError(
                 "the rlm harness has a fixed tool set (ipython) and does not support "
-                "`disabled_tools`; use `skills` to enable built-in skills instead."
+                "`disabled_tools`; use `builtin_skills` to enable built-in skills instead."
             )
         return self
 
@@ -71,8 +72,11 @@ class RLMHarnessConfig(HarnessConfig):
 class RLMHarness(Harness[RLMHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = True
+    SUPPORTS_SKILLS = True
 
     async def setup(self, runtime: Runtime) -> None:
+        # Before the installer: install.sh packages the skills it finds.
+        await self.install_skills(runtime, SKILLS_DIR)
         # install.sh fetches curl/uv itself; add git only when the image lacks it.
         install = (
             "command -v git >/dev/null 2>&1 || "
@@ -123,8 +127,8 @@ class RLMHarness(Harness[RLMHarnessConfig]):
         }
         if system_prompt is not None:
             env["RLM_APPEND_TO_SYSTEM_PROMPT"] = system_prompt
-        if self.config.skills:
-            env["RLM_SKILLS"] = ",".join(self.config.skills)
+        if self.config.builtin_skills:
+            env["RLM_SKILLS"] = ",".join(self.config.builtin_skills)
         if mcp_urls:
             env["RLM_MCP_CONFIG"] = json.dumps(
                 {"mcpServers": {name: {"url": url} for name, url in mcp_urls.items()}}

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/harnesses/terminus_2/harness.py

@@ -18,18 +18,10 @@ class Terminus2HarnessConfig(HarnessConfig):
 class Terminus2Harness(Harness[Terminus2HarnessConfig]):
     APPENDS_SYSTEM_PROMPT = True
     SUPPORTS_MCP = False
+    # Beyond the usual host leaks, Terminus drives tmux: on the host its tmux server —
+    # and the `tmux kill-server` cleanup in `launch` — would share the user's own.
 
     async def setup(self, runtime: Runtime) -> None:
-        # TODO: Terminus drives tmux; on the host (subprocess) runtime its tmux server — and the
-        # `tmux kill-server` cleanup in `launch` — share the host's tmux, so a host run can kill
-        # the user's own tmux session. Until tmux is isolated (a dedicated `tmux -L` socket + a
-        # created, private TMUX_TMPDIR), refuse the host runtime; run Terminus 2 in a container.
-        if runtime.type == "subprocess":
-            raise RuntimeError(
-                "Terminus 2 drives tmux and is unsafe on the subprocess (host) runtime — its tmux "
-                "cleanup can kill the host's tmux server. Run it in a container runtime "
-                "(--env.agent.harness.runtime.type docker|prime|modal)."
-            )
         source = PROGRAM_SOURCE.replace("{version}", self.config.version)
         await runtime.prepare_uv_script(source, self.config.resolved_env)
 

{verifiers-0.2.2.dev17 → verifiers-0.2.2.dev18}/verifiers/v1/utils/compile.py

@@ -95,6 +95,18 @@ def validate_pairing(
             f"{task_cls.__name__} exposes tool servers (MCP). Run it with a harness that "
             f"supports MCP (e.g. --env.agent.harness.id bash), or use tasks without tools."
         )
+    if not harness.SUPPORTS_SKILLS and harness.config.skills:
+        raise ValueError(
+            f"Harness {harness.config.id!r} has no native skill support, but "
+            "`skills` is set. Run them with a harness whose program discovers "
+            "skills (e.g. --env.agent.harness.id claude-code)."
+        )
+    if harness.NEEDS_CONTAINER and isinstance(runtime_config, SubprocessConfig):
+        raise ValueError(
+            f"Harness {harness.config.id!r} needs a container runtime "
+            "(NEEDS_CONTAINER), but this run resolves to the subprocess runtime; "
+            "use --env.agent.harness.runtime.type docker or prime."
+        )
     if not harness.SUPPORTS_USER_SIM and task_cls.user is not None:
         raise ValueError(
             f"Harness {harness.config.id!r} does not drive a user simulator, but "