verifiers 0.2.2.dev58__py3-none-any.whl → 0.2.2.dev60__py3-none-any.whl

verifiers/v1/agent.py

@@ -25,7 +25,7 @@ from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
 from verifiers.v1.retries import backoff, trace_should_retry
-from verifiers.v1.rollout import RolloutRun, _as_messages
+from verifiers.v1.rollout import RolloutRun, RolloutTimeouts, _as_messages
 from verifiers.v1.runtimes import (
     NetworkPolicyConfig,
     Runtime,
@@ -533,23 +533,23 @@ class Agent:
             "harness": self.harness,
             "ctx": self.ctx,
             "runtime_config": runtime_config,
-            "setup_timeout": (
-                self.timeout.setup
-                if self.timeout.setup is not None
-                else task.data.timeout.setup
-            ),
-            "agent_timeout": cap_remote_agent_timeout(
-                agent_timeout, runtime_config, task
-            ),
-            "finalize_timeout": (
-                self.timeout.finalize
-                if self.timeout.finalize is not None
-                else task.data.timeout.finalize
-            ),
-            "scoring_timeout": (
-                self.timeout.scoring
-                if self.timeout.scoring is not None
-                else task.data.timeout.scoring
+            "timeouts": RolloutTimeouts(
+                setup=(
+                    self.timeout.setup
+                    if self.timeout.setup is not None
+                    else task.data.timeout.setup
+                ),
+                agent=cap_remote_agent_timeout(agent_timeout, runtime_config, task),
+                finalize=(
+                    self.timeout.finalize
+                    if self.timeout.finalize is not None
+                    else task.data.timeout.finalize
+                ),
+                scoring=(
+                    self.timeout.scoring
+                    if self.timeout.scoring is not None
+                    else task.data.timeout.scoring
+                ),
             ),
             "limits": self.limits,
             "shared_tools": shared_tools,

verifiers/v1/harnesses/__init__.py

@@ -8,6 +8,10 @@ from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarnessConfig,
 )
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
+from verifiers.v1.harnesses.hermes_agent import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
 from verifiers.v1.harnesses.mini_swe_agent import (
     MiniSWEAgentHarness,
@@ -28,6 +32,8 @@ __all__ = [
     "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
+    "HermesAgentHarness",
+    "HermesAgentHarnessConfig",
     "KimiCodeHarness",
     "KimiCodeHarnessConfig",
     "MiniSWEAgentHarness",

verifiers/v1/harnesses/hermes_agent/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.hermes_agent.harness import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
+
+__all__ = ["HermesAgentHarness", "HermesAgentHarnessConfig"]

verifiers/v1/harnesses/hermes_agent/harness.py

@@ -0,0 +1,140 @@
+"""Run Hermes Agent against interception through its native ACP server."""
+
+import json
+import logging
+from pathlib import Path
+
+from pydantic import Field
+
+from verifiers.v1.acp import ACP
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
+SKILLS_DIR = ".vf-hermes-skills"
+HERMES_ACP = ACP()
+PROVIDER = "intercept"
+logger = logging.getLogger(__name__)
+
+
+class HermesAgentHarnessConfig(HarnessConfig):
+    version: str = Field(default="0.19.0", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Hermes Agent release to install, pinned for reproducibility."""
+    use_bundled_skill: bool = False
+    """Enable Hermes Agent's bundled skill catalog in addition to uploaded skills."""
+
+
+class HermesAgentHarness(Harness[HermesAgentHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_RESUME = True
+    SUPPORTS_SKILLS = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
+        logger.info(
+            "hermes-agent: ensuring Hermes Agent %s is installed", self.config.version
+        )
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
+        await HERMES_ACP.setup(self, runtime)
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ProgramResult:
+        if self.config.disabled_tools:
+            raise ValueError("Hermes Agent ACP does not support disabling tools")
+
+        system_prompt, prompt = self.resolve_prompt(data)
+        home = self._home(trace)
+        model: dict[str, object] = {
+            "provider": PROVIDER,
+            "default": ctx.model,
+            "base_url": endpoint,
+            "api_mode": "chat_completions",
+        }
+        if ctx.sampling.max_tokens is not None:
+            model["max_tokens"] = ctx.sampling.max_tokens
+        await runtime.write(
+            f"{home}/config.yaml",
+            json.dumps(
+                {
+                    "model": model,
+                    # The ACP client already approves tool requests. Avoid routing Hermes'
+                    # redundant smart-approval model calls through interception as turns.
+                    "approvals": {"mode": "off"},
+                    "providers": {
+                        PROVIDER: {
+                            "api": endpoint,
+                            "api_key": secret,
+                            "transport": "chat_completions",
+                            "discover_models": False,
+                            "models": [ctx.model],
+                        }
+                    },
+                }
+            ).encode(),
+        )
+        if not self.config.use_bundled_skill:
+            await runtime.write(
+                f"{home}/.no-bundled-skills",
+                b"Verifiers supplies evaluation skills explicitly.\n",
+            )
+
+        if self.config.skills:
+            skill_home = f"{home}/skills"
+            for command in (
+                ["rm", "-rf", skill_home],
+                ["cp", "-R", SKILLS_DIR, skill_home],
+            ):
+                copied = await runtime.run(command, {})
+                if copied.exit_code != 0:
+                    raise RuntimeError(
+                        f"failed to stage Hermes skills: {copied.stderr.strip()[-500:]}"
+                    )
+
+        env = {
+            **self.config.resolved_env,
+            "HERMES_HOME": home,
+            "HERMES_ACP_SKIP_CONFIGURED_MCP": "1",
+        }
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        command = await runtime.prepare_uv_script(source, env)
+        calls_before = len(trace.calls)
+        result = await HERMES_ACP.run(
+            runtime,
+            env,
+            command,
+            prompt,
+            mcp_urls=mcp_urls,
+            system_prompt=system_prompt,
+            session_path=f"{home}/acp-session",
+        )
+        if not any(call.node is not None for call in trace.calls[calls_before:]):
+            detail = (result.stderr or result.stdout).strip()[-500:] or "<no output>"
+            raise RuntimeError(
+                "Hermes Agent completed without committing a model turn: " + detail
+            )
+        return result
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        result = await runtime.run(["rm", "-rf", self._home(trace)], {})
+        if result.exit_code != 0:
+            raise RuntimeError(
+                f"failed to clean up Hermes home: {result.stderr.strip()[-500:]}"
+            )
+
+    @staticmethod
+    def _home(trace: Trace) -> str:
+        return f"/tmp/vf-hermes/{trace.id}"

verifiers/v1/harnesses/hermes_agent/program.py

@@ -0,0 +1,10 @@
+# /// script
+# requires-python = ">=3.11,<3.14"
+# dependencies = ["hermes-agent[acp,mcp]=={version}"]
+# ///
+"""Start Hermes Agent's native ACP server."""
+
+from acp_adapter.entry import main
+
+if __name__ == "__main__":
+    main()

verifiers/v1/rollout.py

@@ -23,6 +23,7 @@ import logging
 import time
 from collections.abc import AsyncIterator, Callable
 from contextlib import AsyncExitStack, asynccontextmanager
+from dataclasses import dataclass
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.agent import AgentConfig
@@ -57,6 +58,19 @@ from verifiers.v1.types import Messages
 logger = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class RolloutTimeouts:
+    """Per-stage rollout timeouts in seconds (None = unlimited), each bounding one
+    lifecycle stage: `setup` covers task + harness setup in `open()`, `agent` is the
+    cumulative budget the run's segments draw down across `step()`s, `finalize` and
+    `scoring` bound their `close()` stages."""
+
+    setup: float | None = None
+    agent: float | None = None
+    finalize: float | None = None
+    scoring: float | None = None
+
+
 def _as_messages(raw: Messages) -> Messages:
     """A turn's messages may arrive typed or as wire dicts (env code naturally
     writes `{"role": "user", ...}`); the trace speaks typed, so normalize here."""
@@ -115,10 +129,7 @@ class RolloutRun:
         runtime_config: RuntimeConfig,
         wire_data: TaskData | None = None,
         has_user: bool = False,
-        setup_timeout: float | None = None,
-        agent_timeout: float | None = None,
-        finalize_timeout: float | None = None,
-        scoring_timeout: float | None = None,
+        timeouts: RolloutTimeouts | None = None,
         limits: RolloutLimits | None = None,
         shared_tools: dict[str, SharedToolServer] | None = None,
         interception: Interception | None = None,
@@ -130,11 +141,8 @@ class RolloutRun:
         self.ctx = ctx
         self.runtime_config = runtime_config
         self._has_user = has_user
-        self._setup_timeout = setup_timeout
-        self._agent_timeout = agent_timeout
-        self._agent_time_remaining = agent_timeout
-        self._finalize_timeout = finalize_timeout
-        self._scoring_timeout = scoring_timeout
+        self._timeouts = timeouts or RolloutTimeouts()
+        self._agent_time_remaining = self._timeouts.agent
         self._shared_tools = shared_tools or {}
         self._interception = interception
         self.runtime = runtime
@@ -164,7 +172,7 @@ class RolloutRun:
         self.deadline_at: float | None = None
         """The active harness segment's absolute deadline (event-loop clock), or
         None between segments / when unbounded. An interaction spends one cumulative
-        `agent_timeout` budget only while its own segments run, so time awaiting
+        `timeouts.agent` budget only while its own segments run, so time awaiting
         the caller (including another interleaved agent) cannot starve it."""
 
     @property
@@ -244,8 +252,8 @@ class RolloutRun:
             # Task setup and harness provisioning share one setup-stage deadline.
             setup_deadline = (
                 None
-                if self._setup_timeout is None
-                else asyncio.get_running_loop().time() + self._setup_timeout
+                if self._timeouts.setup is None
+                else asyncio.get_running_loop().time() + self._timeouts.setup
             )
             async with (
                 boundary(TaskError, "task setup"),
@@ -340,7 +348,7 @@ class RolloutRun:
                 self.fail(
                     HarnessError(
                         f"agent timeout: rollout exceeded its "
-                        f"{self._agent_timeout:g}s budget"
+                        f"{self._timeouts.agent:g}s budget"
                     )
                 )
             else:
@@ -406,7 +414,7 @@ class RolloutRun:
                         invoke(
                             self.task.finalize, {"trace": trace, "runtime": runtime}
                         ),
-                        self._finalize_timeout,
+                        self._timeouts.finalize,
                     )
                 now = time.time()
                 trace.timing.finalize.end = now
@@ -418,7 +426,7 @@ class RolloutRun:
                             self.task.score(trace, runtime),
                             self.harness.score(trace, runtime),
                         ),
-                        self._scoring_timeout,
+                        self._timeouts.scoring,
                     )
                 trace.timing.scoring.end = time.time()
         except Exception as e:  # noqa: BLE001 - finalize boundary records every rollout failure

{verifiers-0.2.2.dev58.dist-info → verifiers-0.2.2.dev60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev58
+Version: 0.2.2.dev60
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev58.dist-info → verifiers-0.2.2.dev60.dist-info}/RECORD

@@ -168,7 +168,7 @@ verifiers/utils/tool_utils.py,sha256=gInWZODWQUZN4TyEMuuITyOUm9qlHJxtcvr1zZl_zJs
 verifiers/utils/usage_utils.py,sha256=Mw6WC78r6EsLyRFdP9ICveqptkjizbSkAC86ZH0geVU,3918
 verifiers/utils/version_utils.py,sha256=39gV8-mNF9GspAgKloLo8oy7_MVluMrwGPrntQv2hx4,2656
 verifiers/v1/__init__.py,sha256=Im5qTZy9mH6PRTUi-1V6W7EWKZzJyTWLGKMYALbWz_w,7467
-verifiers/v1/agent.py,sha256=t5Z_iI6Zyt0ieMDI7pfHUVYIKViuSXF-CI_rIApx3f4,32071
+verifiers/v1/agent.py,sha256=ASbe6IHAYcwivzQkHunX3cGu-o10Gez8tlVhO-pLYyE,32134
 verifiers/v1/artifacts.py,sha256=5eKOjDSUWwj7GrhxILMHdz2ngEONWMdcTGlBcuGgrLU,5834
 verifiers/v1/decorators.py,sha256=9CcCwgZlRjVGPqT5G1CFJUHP4OIMuBGkUJZkcI6dnbk,3865
 verifiers/v1/env.py,sha256=TGCq_rPp_VdyGAKpHuscSDyIxIGnim2N-dpKkG-J5p0,18175
@@ -181,7 +181,7 @@ verifiers/v1/legacy.py,sha256=ISk3Bix9Hy5EjZtyV9QO0LNyDNDklBHVjFokOrw_3U8,22628
 verifiers/v1/loaders.py,sha256=NSIBdNU3JVeyddL-dM4dJt6khCHyjCgzJuFtLrgiwD0,9989
 verifiers/v1/push.py,sha256=MWZmvyj0iyzNIO0xRhlkCXtFF1plq6EJi8rWlvrFtvI,11363
 verifiers/v1/retries.py,sha256=quUGVruUkZx-9WQcFiocMfL_M056voU8gwDqsuLR3f4,5412
-verifiers/v1/rollout.py,sha256=ehPoN6EnrIg_lvF1-pdf9pS8j47K3z6jDHc66MixzX0,20470
+verifiers/v1/rollout.py,sha256=s8ji1W19FHZOn-Wn3_fjw-b1hhmi0EqZ7dZVntYK5pM,20732
 verifiers/v1/scoring.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/session.py,sha256=J6ZPuvsg6vMTJAs9dMUMtfdjCuvp6Hom877DVdKFKlY,6912
 verifiers/v1/state.py,sha256=pcpN2V6tX7rIO5XdtdUYay1j0ktEwrMtq7w1ApfsPdE,675
@@ -252,7 +252,7 @@ verifiers/v1/gepa/config.py,sha256=gFn8Re4GoeHJs2PZ-iw4jCEZ72AiM1lNgwaA0uvJlYw,4
 verifiers/v1/gepa/dataset.py,sha256=N2QYqUljHVuAdCbUuydEIlarvAWa6LFdGKu9qsOvjro,2186
 verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
 verifiers/v1/gepa/runner.py,sha256=-LZEt0lod6NKfyxg2dvOXZjfzTv3AWwendexd1De26c,6260
-verifiers/v1/harnesses/__init__.py,sha256=706ES37vrh_kCbctnbgkXIb1jBtO2ozZY5klBSg8ss0,1460
+verifiers/v1/harnesses/__init__.py,sha256=CDKLCWYqRbVy3NvEzavC12_DaxuoHQfw9AQ8hmlQuBQ,1624
 verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
 verifiers/v1/harnesses/bash/harness.py,sha256=ZtjhsANuK3vXkf9mRCi8-sBMyPebJMBNHa58vTd2Fm8,5284
 verifiers/v1/harnesses/bash/program.py,sha256=hGv3k9jwoabL045AGiZWiX76Cca3Juk58AaMYeJTEGo,15334
@@ -263,6 +263,9 @@ verifiers/v1/harnesses/claude_code/__init__.py,sha256=JkZQylMSA3o1fvBx_NM41y_umu
 verifiers/v1/harnesses/claude_code/harness.py,sha256=98k5T8d8nFQmxIU-ytkrkrrP6UNR5NxoGhVJRLVa-fI,4015
 verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
 verifiers/v1/harnesses/codex/harness.py,sha256=OG1riQdVmZzVpHENsd0UzsIbdIAxDMHgRikmo73HL5M,13071
+verifiers/v1/harnesses/hermes_agent/__init__.py,sha256=k84Y8UTTf1KOw5LGqPb2dk-hYFagPevDJBOF7qsR_vo,176
+verifiers/v1/harnesses/hermes_agent/harness.py,sha256=AveZi9g4pga9pYy4VAmn_KMKlJw3KhwG45JZcPxbdR4,5094
+verifiers/v1/harnesses/hermes_agent/program.py,sha256=9qPLbUnhox2bGu5ypEEnBhw9QKtYwNvrbl7W_zeNMdk,229
 verifiers/v1/harnesses/kimi_code/__init__.py,sha256=1BEotSfuzmVcE7UrxHeAREM5d7J20cGkonHuowlrzu4,161
 verifiers/v1/harnesses/kimi_code/harness.py,sha256=Pc2uiBsia-uQFuXvaLMFdblE6TS-p0E2cvB6ofRrHK0,4515
 verifiers/v1/harnesses/mini_swe_agent/__init__.py,sha256=1OBu2Y-XqL72-dGhFdgB6sr5uX_peEEXT9vEjPy1ovU,182
@@ -333,8 +336,8 @@ verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/sampling.py,sha256=52TJ5-Hmpvp4YSgKNR-LmAPVwm4RDcH-o2-BnbVTmYQ,965
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev58.dist-info/METADATA,sha256=dIolspvil7G19vdj_DgeI_K_JX0sxuOxXcBT1DG6-_Q,4545
-verifiers-0.2.2.dev58.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev58.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev58.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev58.dist-info/RECORD,,
+verifiers-0.2.2.dev60.dist-info/METADATA,sha256=aVyGlQcJ7KN2Qf6QMshBZTNvO6KN9rMYQNqpExl1EeY,4545
+verifiers-0.2.2.dev60.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev60.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev60.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev60.dist-info/RECORD,,