verifiers 0.2.2.dev58__tar.gz → 0.2.2.dev60__tar.gz

{verifiers-0.2.2.dev58 → verifiers-0.2.2.dev60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev58
+Version: 0.2.2.dev60
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev58 → verifiers-0.2.2.dev60}/tests/conftest.py

@@ -45,6 +45,9 @@ def pytest_configure(config) -> None:
     config.addinivalue_line(
         "markers", "claude_code: v1 e2e cases on the claude-code harness"
     )
+    config.addinivalue_line(
+        "markers", "hermes_agent: v1 e2e cases on the hermes-agent harness"
+    )
 
 
 @pytest.fixture

{verifiers-0.2.2.dev58 → verifiers-0.2.2.dev60}/tests/v1/conftest.py

@@ -25,7 +25,8 @@ Every combination carries its axes' pytest marks, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
-harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code`.
+harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `pool` / `codex` / `claude_code` /
+`hermes_agent`.
 A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
 combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
 prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
@@ -69,8 +70,8 @@ def tool_runtime(request) -> dict:
 
 
 # Built-in harnesses are bundled in the `harnesses` package; the agent CLIs (`rlm` /
-# `kimi-code` / `codex` / `claude-code`) install their dependencies at rollout. `compact` (an
-# example harness) and `terminus-2` (drives the host tmux) are excluded from e2e.
+# `kimi-code` / `codex` / `claude-code` / `hermes-agent`) install their dependencies at rollout.
+# `compact` (an example harness) and `terminus-2` (drives the host tmux) are excluded from e2e.
 @pytest.fixture
 def harness(request) -> str:
     return request.param

{verifiers-0.2.2.dev58 → verifiers-0.2.2.dev60}/tests/v1/test_e2e.py

@@ -37,6 +37,7 @@ AGENTIC_PLACEMENTS = [
     _pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
     _pair("codex", "docker", "codex-harness-in-docker"),
     _pair("claude-code", "docker", "claude-code-harness-in-docker"),
+    _pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
     _pair("bash", "prime", "bash-harness-in-prime"),
     _pair("bash", "modal", "bash-harness-in-modal"),
 ]
@@ -54,6 +55,7 @@ USER_RUNTIMES = [
 # retain MCP access after resuming. Cover every harness in the local container runtime,
 # plus one remote placement for the sandbox/tunnel boundary.
 ACP_RESUME_PLACEMENTS = [
+    _pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
     _pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
     _pair("pi", "docker", "pi-acp-in-docker"),
     _pair("pool", "docker", "pool-acp-in-docker"),

{verifiers-0.2.2.dev58 → verifiers-0.2.2.dev60}/verifiers/v1/agent.py

@@ -25,7 +25,7 @@ from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
 from verifiers.v1.retries import backoff, trace_should_retry
-from verifiers.v1.rollout import RolloutRun, _as_messages
+from verifiers.v1.rollout import RolloutRun, RolloutTimeouts, _as_messages
 from verifiers.v1.runtimes import (
     NetworkPolicyConfig,
     Runtime,
@@ -533,23 +533,23 @@ class Agent:
             "harness": self.harness,
             "ctx": self.ctx,
             "runtime_config": runtime_config,
-            "setup_timeout": (
-                self.timeout.setup
-                if self.timeout.setup is not None
-                else task.data.timeout.setup
-            ),
-            "agent_timeout": cap_remote_agent_timeout(
-                agent_timeout, runtime_config, task
-            ),
-            "finalize_timeout": (
-                self.timeout.finalize
-                if self.timeout.finalize is not None
-                else task.data.timeout.finalize
-            ),
-            "scoring_timeout": (
-                self.timeout.scoring
-                if self.timeout.scoring is not None
-                else task.data.timeout.scoring
+            "timeouts": RolloutTimeouts(
+                setup=(
+                    self.timeout.setup
+                    if self.timeout.setup is not None
+                    else task.data.timeout.setup
+                ),
+                agent=cap_remote_agent_timeout(agent_timeout, runtime_config, task),
+                finalize=(
+                    self.timeout.finalize
+                    if self.timeout.finalize is not None
+                    else task.data.timeout.finalize
+                ),
+                scoring=(
+                    self.timeout.scoring
+                    if self.timeout.scoring is not None
+                    else task.data.timeout.scoring
+                ),
             ),
             "limits": self.limits,
             "shared_tools": shared_tools,

{verifiers-0.2.2.dev58 → verifiers-0.2.2.dev60}/verifiers/v1/harnesses/__init__.py

@@ -8,6 +8,10 @@ from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarnessConfig,
 )
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
+from verifiers.v1.harnesses.hermes_agent import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
 from verifiers.v1.harnesses.mini_swe_agent import (
     MiniSWEAgentHarness,
@@ -28,6 +32,8 @@ __all__ = [
     "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
+    "HermesAgentHarness",
+    "HermesAgentHarnessConfig",
     "KimiCodeHarness",
     "KimiCodeHarnessConfig",
     "MiniSWEAgentHarness",

verifiers-0.2.2.dev60/verifiers/v1/harnesses/hermes_agent/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.hermes_agent.harness import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
+
+__all__ = ["HermesAgentHarness", "HermesAgentHarnessConfig"]

verifiers-0.2.2.dev60/verifiers/v1/harnesses/hermes_agent/harness.py

@@ -0,0 +1,140 @@
+"""Run Hermes Agent against interception through its native ACP server."""
+
+import json
+import logging
+from pathlib import Path
+
+from pydantic import Field
+
+from verifiers.v1.acp import ACP
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
+SKILLS_DIR = ".vf-hermes-skills"
+HERMES_ACP = ACP()
+PROVIDER = "intercept"
+logger = logging.getLogger(__name__)
+
+
+class HermesAgentHarnessConfig(HarnessConfig):
+    version: str = Field(default="0.19.0", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Hermes Agent release to install, pinned for reproducibility."""
+    use_bundled_skill: bool = False
+    """Enable Hermes Agent's bundled skill catalog in addition to uploaded skills."""
+
+
+class HermesAgentHarness(Harness[HermesAgentHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_RESUME = True
+    SUPPORTS_SKILLS = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
+        logger.info(
+            "hermes-agent: ensuring Hermes Agent %s is installed", self.config.version
+        )
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
+        await HERMES_ACP.setup(self, runtime)
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ProgramResult:
+        if self.config.disabled_tools:
+            raise ValueError("Hermes Agent ACP does not support disabling tools")
+
+        system_prompt, prompt = self.resolve_prompt(data)
+        home = self._home(trace)
+        model: dict[str, object] = {
+            "provider": PROVIDER,
+            "default": ctx.model,
+            "base_url": endpoint,
+            "api_mode": "chat_completions",
+        }
+        if ctx.sampling.max_tokens is not None:
+            model["max_tokens"] = ctx.sampling.max_tokens
+        await runtime.write(
+            f"{home}/config.yaml",
+            json.dumps(
+                {
+                    "model": model,
+                    # The ACP client already approves tool requests. Avoid routing Hermes'
+                    # redundant smart-approval model calls through interception as turns.
+                    "approvals": {"mode": "off"},
+                    "providers": {
+                        PROVIDER: {
+                            "api": endpoint,
+                            "api_key": secret,
+                            "transport": "chat_completions",
+                            "discover_models": False,
+                            "models": [ctx.model],
+                        }
+                    },
+                }
+            ).encode(),
+        )
+        if not self.config.use_bundled_skill:
+            await runtime.write(
+                f"{home}/.no-bundled-skills",
+                b"Verifiers supplies evaluation skills explicitly.\n",
+            )
+
+        if self.config.skills:
+            skill_home = f"{home}/skills"
+            for command in (
+                ["rm", "-rf", skill_home],
+                ["cp", "-R", SKILLS_DIR, skill_home],
+            ):
+                copied = await runtime.run(command, {})
+                if copied.exit_code != 0:
+                    raise RuntimeError(
+                        f"failed to stage Hermes skills: {copied.stderr.strip()[-500:]}"
+                    )
+
+        env = {
+            **self.config.resolved_env,
+            "HERMES_HOME": home,
+            "HERMES_ACP_SKIP_CONFIGURED_MCP": "1",
+        }
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        command = await runtime.prepare_uv_script(source, env)
+        calls_before = len(trace.calls)
+        result = await HERMES_ACP.run(
+            runtime,
+            env,
+            command,
+            prompt,
+            mcp_urls=mcp_urls,
+            system_prompt=system_prompt,
+            session_path=f"{home}/acp-session",
+        )
+        if not any(call.node is not None for call in trace.calls[calls_before:]):
+            detail = (result.stderr or result.stdout).strip()[-500:] or "<no output>"
+            raise RuntimeError(
+                "Hermes Agent completed without committing a model turn: " + detail
+            )
+        return result
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        result = await runtime.run(["rm", "-rf", self._home(trace)], {})
+        if result.exit_code != 0:
+            raise RuntimeError(
+                f"failed to clean up Hermes home: {result.stderr.strip()[-500:]}"
+            )
+
+    @staticmethod
+    def _home(trace: Trace) -> str:
+        return f"/tmp/vf-hermes/{trace.id}"

verifiers-0.2.2.dev60/verifiers/v1/harnesses/hermes_agent/program.py

@@ -0,0 +1,10 @@
+# /// script
+# requires-python = ">=3.11,<3.14"
+# dependencies = ["hermes-agent[acp,mcp]=={version}"]
+# ///
+"""Start Hermes Agent's native ACP server."""
+
+from acp_adapter.entry import main
+
+if __name__ == "__main__":
+    main()

{verifiers-0.2.2.dev58 → verifiers-0.2.2.dev60}/verifiers/v1/rollout.py

@@ -23,6 +23,7 @@ import logging
 import time
 from collections.abc import AsyncIterator, Callable
 from contextlib import AsyncExitStack, asynccontextmanager
+from dataclasses import dataclass
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.agent import AgentConfig
@@ -57,6 +58,19 @@ from verifiers.v1.types import Messages
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class RolloutTimeouts:
+    """Per-stage rollout timeouts in seconds (None = unlimited), each bounding one
+    lifecycle stage: `setup` covers task + harness setup in `open()`, `agent` is the
+    cumulative budget the run's segments draw down across `step()`s, `finalize` and
+    `scoring` bound their `close()` stages."""
+
+    setup: float | None = None
+    agent: float | None = None
+    finalize: float | None = None
+    scoring: float | None = None
+
+
 def _as_messages(raw: Messages) -> Messages:
     """A turn's messages may arrive typed or as wire dicts (env code naturally
     writes `{"role": "user", ...}`); the trace speaks typed, so normalize here."""
@@ -115,10 +129,7 @@ class RolloutRun:
         runtime_config: RuntimeConfig,
         wire_data: TaskData | None = None,
         has_user: bool = False,
-        setup_timeout: float | None = None,
-        agent_timeout: float | None = None,
-        finalize_timeout: float | None = None,
-        scoring_timeout: float | None = None,
+        timeouts: RolloutTimeouts | None = None,
         limits: RolloutLimits | None = None,
         shared_tools: dict[str, SharedToolServer] | None = None,
         interception: Interception | None = None,
@@ -130,11 +141,8 @@ class RolloutRun:
         self.ctx = ctx
         self.runtime_config = runtime_config
         self._has_user = has_user
-        self._setup_timeout = setup_timeout
-        self._agent_timeout = agent_timeout
-        self._agent_time_remaining = agent_timeout
-        self._finalize_timeout = finalize_timeout
-        self._scoring_timeout = scoring_timeout
+        self._timeouts = timeouts or RolloutTimeouts()
+        self._agent_time_remaining = self._timeouts.agent
         self._shared_tools = shared_tools or {}
         self._interception = interception
         self.runtime = runtime
@@ -164,7 +172,7 @@ class RolloutRun:
         self.deadline_at: float | None = None
         """The active harness segment's absolute deadline (event-loop clock), or
         None between segments / when unbounded. An interaction spends one cumulative
-        `agent_timeout` budget only while its own segments run, so time awaiting
+        `timeouts.agent` budget only while its own segments run, so time awaiting
         the caller (including another interleaved agent) cannot starve it."""
 
     @property
@@ -244,8 +252,8 @@ class RolloutRun:
             # Task setup and harness provisioning share one setup-stage deadline.
             setup_deadline = (
                 None
-                if self._setup_timeout is None
-                else asyncio.get_running_loop().time() + self._setup_timeout
+                if self._timeouts.setup is None
+                else asyncio.get_running_loop().time() + self._timeouts.setup
             )
             async with (
                 boundary(TaskError, "task setup"),
@@ -340,7 +348,7 @@ class RolloutRun:
                 self.fail(
                     HarnessError(
                         f"agent timeout: rollout exceeded its "
-                        f"{self._agent_timeout:g}s budget"
+                        f"{self._timeouts.agent:g}s budget"
                     )
                 )
             else:
@@ -406,7 +414,7 @@ class RolloutRun:
                         invoke(
                             self.task.finalize, {"trace": trace, "runtime": runtime}
                         ),
-                        self._finalize_timeout,
+                        self._timeouts.finalize,
                     )
                 now = time.time()
                 trace.timing.finalize.end = now
@@ -418,7 +426,7 @@ class RolloutRun:
                             self.task.score(trace, runtime),
                             self.harness.score(trace, runtime),
                         ),
-                        self._scoring_timeout,
+                        self._timeouts.scoring,
                     )
                 trace.timing.scoring.end = time.time()
         except Exception as e:  # noqa: BLE001 - finalize boundary records every rollout failure