verifiers 0.2.2.dev57__tar.gz → 0.2.2.dev59__tar.gz

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev57
+Version: 0.2.2.dev59
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/tests/conftest.py

@@ -45,6 +45,9 @@ def pytest_configure(config) -> None:
     config.addinivalue_line(
         "markers", "claude_code: v1 e2e cases on the claude-code harness"
     )
+    config.addinivalue_line(
+        "markers", "hermes_agent: v1 e2e cases on the hermes-agent harness"
+    )
 
 
 @pytest.fixture

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/tests/v1/conftest.py

@@ -25,7 +25,8 @@ Every combination carries its axes' pytest marks, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
-harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code`.
+harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code` /
+`hermes_agent`.
 A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
 combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
 prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
@@ -69,8 +70,8 @@ def tool_runtime(request) -> dict:
 
 
 # Built-in harnesses are bundled in the `harnesses` package; the agent CLIs (`rlm` /
-# `kimi-code` / `codex` / `claude-code`) install their dependencies at rollout. `compact` (an
-# example harness) and `terminus-2` (drives the host tmux) are excluded from e2e.
+# `kimi-code` / `codex` / `claude-code` / `hermes-agent`) install their dependencies at rollout.
+# `compact` (an example harness) and `terminus-2` (drives the host tmux) are excluded from e2e.
 @pytest.fixture
 def harness(request) -> str:
     return request.param

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/tests/v1/fixtures/echo_tool_v1.py

@@ -1,12 +1,13 @@
 """echo (v1, MCP tool): retrieve a stamped echo from a `vf.Toolset`, then report it.
 
-The v1 tool fixture for the e2e matrix. The task declares an `EchoToolset` (`vf.Toolset`)
-with one `@vf.tool` method whose placement is CLI-tunable (`--taskset.task.tools.colocated`,
-`--taskset.task.tools.runtime.type`): it runs colocated in the harness's runtime or in its
-own runtime, and the harness must reach it wherever it lives. The tool stamps its output
-with a token the prompt never reveals, so the reward is 1.0 only if the model actually
-called the tool — trivial when the infra works, impossible when it doesn't. The tool is
-task-agnostic, so it would also serve taskset-scoped (`Taskset.tools`).
+The v1 tool fixture for the e2e matrix. The task constructs an `EchoToolset`
+(`vf.Toolset`) in `Task.toolsets`, with one `@vf.tool` method whose placement is
+CLI-tunable (`--taskset.task.tools.colocated`, `--taskset.task.tools.runtime.type`):
+it runs colocated in the harness's runtime or in its own runtime, and the harness
+must reach it wherever it lives. The tool stamps its output with a token the prompt
+never reveals, so the reward is 1.0 only if the model actually called the tool —
+trivial when the infra works, impossible when it doesn't. The tool is task-agnostic,
+so it would also serve taskset-scoped (`Taskset.toolsets`).
 """
 
 import verifiers.v1 as vf

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/tests/v1/test_e2e.py

@@ -37,6 +37,7 @@ AGENTIC_PLACEMENTS = [
     _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("codex", "docker", "codex-harness-in-docker"),
     _pair("claude-code", "docker", "claude-code-harness-in-docker"),
+    _pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]
@@ -54,6 +55,7 @@ USER_RUNTIMES = [
 # retain MCP access after resuming. Cover every harness in the local container runtime,
 # plus one remote placement for the sandbox/tunnel boundary.
 ACP_RESUME_PLACEMENTS = [
+    _pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
     _pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
     _pair("pi", "docker", "pi-acp-in-docker"),
     _pair("pool", "docker", "pool-acp-in-docker"),
@@ -234,7 +236,7 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     the harness's runtime, or its own runtime) x the harness `runtime`. The tool stamps
     its output with a token the prompt never reveals, so reward 1.0 proves the tool was
     reachable from wherever the harness runs and actually ran. Eval-wide SHARED servers
-    are a different scope (`Taskset.tools`) with their own env-server-path coverage:
+    are a different scope (`Taskset.toolsets`) with their own env-server-path coverage:
     `test_shared_tool_isolation`."""
     (trace,) = await run_v1(
         "echo-tool-v1",

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/verifiers/v1/env.py

@@ -379,7 +379,7 @@ class Env(ABC, Generic[ConfigT]):
     def _requires_tunnel(self, shared: dict[str, SharedToolServer]) -> bool:
         """`requires_tunnel` over the consumers known before any rollout: role
         runtimes, live `shared` servers, and the task class's tool servers
-        (their configs read off the declared `tools` fields, no task needed)."""
+        (`Task.toolsets` is a classmethod, so no task instance is needed)."""
         task_cls = type(self.taskset).task_type()
         configs = [
             server.config for server in task_cls.toolsets(self.taskset.config.task)

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/verifiers/v1/harnesses/__init__.py

@@ -8,6 +8,10 @@ from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarnessConfig,
 )
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
+from verifiers.v1.harnesses.hermes_agent import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
 from verifiers.v1.harnesses.mini_swe_agent import (
     MiniSWEAgentHarness,
@@ -28,6 +32,8 @@ __all__ = [
     "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
+    "HermesAgentHarness",
+    "HermesAgentHarnessConfig",
     "KimiCodeHarness",
     "KimiCodeHarnessConfig",
     "MiniSWEAgentHarness",

verifiers-0.2.2.dev59/verifiers/v1/harnesses/hermes_agent/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.hermes_agent.harness import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
+
+__all__ = ["HermesAgentHarness", "HermesAgentHarnessConfig"]

verifiers-0.2.2.dev59/verifiers/v1/harnesses/hermes_agent/harness.py

@@ -0,0 +1,140 @@
+"""Run Hermes Agent against interception through its native ACP server."""
+
+import json
+import logging
+from pathlib import Path
+
+from pydantic import Field
+
+from verifiers.v1.acp import ACP
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
+SKILLS_DIR = ".vf-hermes-skills"
+HERMES_ACP = ACP()
+PROVIDER = "intercept"
+logger = logging.getLogger(__name__)
+
+
+class HermesAgentHarnessConfig(HarnessConfig):
+    version: str = Field(default="0.19.0", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Hermes Agent release to install, pinned for reproducibility."""
+    use_bundled_skill: bool = False
+    """Enable Hermes Agent's bundled skill catalog in addition to uploaded skills."""
+
+
+class HermesAgentHarness(Harness[HermesAgentHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_RESUME = True
+    SUPPORTS_SKILLS = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
+        logger.info(
+            "hermes-agent: ensuring Hermes Agent %s is installed", self.config.version
+        )
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
+        await HERMES_ACP.setup(self, runtime)
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ProgramResult:
+        if self.config.disabled_tools:
+            raise ValueError("Hermes Agent ACP does not support disabling tools")
+
+        system_prompt, prompt = self.resolve_prompt(data)
+        home = self._home(trace)
+        model: dict[str, object] = {
+            "provider": PROVIDER,
+            "default": ctx.model,
+            "base_url": endpoint,
+            "api_mode": "chat_completions",
+        }
+        if ctx.sampling.max_tokens is not None:
+            model["max_tokens"] = ctx.sampling.max_tokens
+        await runtime.write(
+            f"{home}/config.yaml",
+            json.dumps(
+                {
+                    "model": model,
+                    # The ACP client already approves tool requests. Avoid routing Hermes'
+                    # redundant smart-approval model calls through interception as turns.
+                    "approvals": {"mode": "off"},
+                    "providers": {
+                        PROVIDER: {
+                            "api": endpoint,
+                            "api_key": secret,
+                            "transport": "chat_completions",
+                            "discover_models": False,
+                            "models": [ctx.model],
+                        }
+                    },
+                }
+            ).encode(),
+        )
+        if not self.config.use_bundled_skill:
+            await runtime.write(
+                f"{home}/.no-bundled-skills",
+                b"Verifiers supplies evaluation skills explicitly.\n",
+            )
+
+        if self.config.skills:
+            skill_home = f"{home}/skills"
+            for command in (
+                ["rm", "-rf", skill_home],
+                ["cp", "-R", SKILLS_DIR, skill_home],
+            ):
+                copied = await runtime.run(command, {})
+                if copied.exit_code != 0:
+                    raise RuntimeError(
+                        f"failed to stage Hermes skills: {copied.stderr.strip()[-500:]}"
+                    )
+
+        env = {
+            **self.config.resolved_env,
+            "HERMES_HOME": home,
+            "HERMES_ACP_SKIP_CONFIGURED_MCP": "1",
+        }
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        command = await runtime.prepare_uv_script(source, env)
+        calls_before = len(trace.calls)
+        result = await HERMES_ACP.run(
+            runtime,
+            env,
+            command,
+            prompt,
+            mcp_urls=mcp_urls,
+            system_prompt=system_prompt,
+            session_path=f"{home}/acp-session",
+        )
+        if not any(call.node is not None for call in trace.calls[calls_before:]):
+            detail = (result.stderr or result.stdout).strip()[-500:] or "<no output>"
+            raise RuntimeError(
+                "Hermes Agent completed without committing a model turn: " + detail
+            )
+        return result
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        result = await runtime.run(["rm", "-rf", self._home(trace)], {})
+        if result.exit_code != 0:
+            raise RuntimeError(
+                f"failed to clean up Hermes home: {result.stderr.strip()[-500:]}"
+            )
+
+    @staticmethod
+    def _home(trace: Trace) -> str:
+        return f"/tmp/vf-hermes/{trace.id}"

verifiers-0.2.2.dev59/verifiers/v1/harnesses/hermes_agent/program.py

@@ -0,0 +1,10 @@
+# /// script
+# requires-python = ">=3.11,<3.14"
+# dependencies = ["hermes-agent[acp,mcp]=={version}"]
+# ///
+"""Start Hermes Agent's native ACP server."""
+
+from acp_adapter.entry import main
+
+if __name__ == "__main__":
+    main()

{verifiers-0.2.2.dev57 → verifiers-0.2.2.dev59}/verifiers/v1/task.py

@@ -3,8 +3,8 @@
 `TaskData` is the wire half: a frozen pydantic model carrying the data which
 initializes a task instance. Rides on `trace.task.data` in `traces.jsonl`.
 
-`Task` is the behavior half: runtime prep (`setup`/`finalize`), tool declarations
-(`tools`), and scoring (`@reward`/`@metric`) methods.
+`Task` is the behavior half: runtime prep (`setup`/`finalize`), tool servers
+(`toolsets`), and scoring (`@reward`/`@metric`) methods.
 """
 
 from __future__ import annotations