verifiers 0.2.2.dev57__py3-none-any.whl → 0.2.2.dev59__py3-none-any.whl

verifiers/v1/env.py

@@ -379,7 +379,7 @@ class Env(ABC, Generic[ConfigT]):
     def _requires_tunnel(self, shared: dict[str, SharedToolServer]) -> bool:
         """`requires_tunnel` over the consumers known before any rollout: role
         runtimes, live `shared` servers, and the task class's tool servers
-        (their configs read off the declared `tools` fields, no task needed)."""
+        (`Task.toolsets` is a classmethod, so no task instance is needed)."""
         task_cls = type(self.taskset).task_type()
         configs = [
             server.config for server in task_cls.toolsets(self.taskset.config.task)

verifiers/v1/harnesses/__init__.py

@@ -8,6 +8,10 @@ from verifiers.v1.harnesses.claude_code import (
     ClaudeCodeHarnessConfig,
 )
 from verifiers.v1.harnesses.codex import CodexHarness, CodexHarnessConfig
+from verifiers.v1.harnesses.hermes_agent import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
 from verifiers.v1.harnesses.kimi_code import KimiCodeHarness, KimiCodeHarnessConfig
 from verifiers.v1.harnesses.mini_swe_agent import (
     MiniSWEAgentHarness,
@@ -28,6 +32,8 @@ __all__ = [
     "ClaudeCodeHarnessConfig",
     "CodexHarness",
     "CodexHarnessConfig",
+    "HermesAgentHarness",
+    "HermesAgentHarnessConfig",
     "KimiCodeHarness",
     "KimiCodeHarnessConfig",
     "MiniSWEAgentHarness",

verifiers/v1/harnesses/hermes_agent/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.hermes_agent.harness import (
+    HermesAgentHarness,
+    HermesAgentHarnessConfig,
+)
+
+__all__ = ["HermesAgentHarness", "HermesAgentHarnessConfig"]

verifiers/v1/harnesses/hermes_agent/harness.py

@@ -0,0 +1,140 @@
+"""Run Hermes Agent against interception through its native ACP server."""
+
+import json
+import logging
+from pathlib import Path
+
+from pydantic import Field
+
+from verifiers.v1.acp import ACP
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
+SKILLS_DIR = ".vf-hermes-skills"
+HERMES_ACP = ACP()
+PROVIDER = "intercept"
+logger = logging.getLogger(__name__)
+
+
+class HermesAgentHarnessConfig(HarnessConfig):
+    version: str = Field(default="0.19.0", pattern=r"^[A-Za-z0-9._+-]+$")
+    """Hermes Agent release to install, pinned for reproducibility."""
+    use_bundled_skill: bool = False
+    """Enable Hermes Agent's bundled skill catalog in addition to uploaded skills."""
+
+
+class HermesAgentHarness(Harness[HermesAgentHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_RESUME = True
+    SUPPORTS_SKILLS = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await self.install_skills(runtime, SKILLS_DIR)
+        logger.info(
+            "hermes-agent: ensuring Hermes Agent %s is installed", self.config.version
+        )
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        await runtime.prepare_uv_script(source, self.config.resolved_env)
+        await HERMES_ACP.setup(self, runtime)
+
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ProgramResult:
+        if self.config.disabled_tools:
+            raise ValueError("Hermes Agent ACP does not support disabling tools")
+
+        system_prompt, prompt = self.resolve_prompt(data)
+        home = self._home(trace)
+        model: dict[str, object] = {
+            "provider": PROVIDER,
+            "default": ctx.model,
+            "base_url": endpoint,
+            "api_mode": "chat_completions",
+        }
+        if ctx.sampling.max_tokens is not None:
+            model["max_tokens"] = ctx.sampling.max_tokens
+        await runtime.write(
+            f"{home}/config.yaml",
+            json.dumps(
+                {
+                    "model": model,
+                    # The ACP client already approves tool requests. Avoid routing Hermes'
+                    # redundant smart-approval model calls through interception as turns.
+                    "approvals": {"mode": "off"},
+                    "providers": {
+                        PROVIDER: {
+                            "api": endpoint,
+                            "api_key": secret,
+                            "transport": "chat_completions",
+                            "discover_models": False,
+                            "models": [ctx.model],
+                        }
+                    },
+                }
+            ).encode(),
+        )
+        if not self.config.use_bundled_skill:
+            await runtime.write(
+                f"{home}/.no-bundled-skills",
+                b"Verifiers supplies evaluation skills explicitly.\n",
+            )
+
+        if self.config.skills:
+            skill_home = f"{home}/skills"
+            for command in (
+                ["rm", "-rf", skill_home],
+                ["cp", "-R", SKILLS_DIR, skill_home],
+            ):
+                copied = await runtime.run(command, {})
+                if copied.exit_code != 0:
+                    raise RuntimeError(
+                        f"failed to stage Hermes skills: {copied.stderr.strip()[-500:]}"
+                    )
+
+        env = {
+            **self.config.resolved_env,
+            "HERMES_HOME": home,
+            "HERMES_ACP_SKIP_CONFIGURED_MCP": "1",
+        }
+        source = PROGRAM_SOURCE.replace("{version}", self.config.version)
+        command = await runtime.prepare_uv_script(source, env)
+        calls_before = len(trace.calls)
+        result = await HERMES_ACP.run(
+            runtime,
+            env,
+            command,
+            prompt,
+            mcp_urls=mcp_urls,
+            system_prompt=system_prompt,
+            session_path=f"{home}/acp-session",
+        )
+        if not any(call.node is not None for call in trace.calls[calls_before:]):
+            detail = (result.stderr or result.stdout).strip()[-500:] or "<no output>"
+            raise RuntimeError(
+                "Hermes Agent completed without committing a model turn: " + detail
+            )
+        return result
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        result = await runtime.run(["rm", "-rf", self._home(trace)], {})
+        if result.exit_code != 0:
+            raise RuntimeError(
+                f"failed to clean up Hermes home: {result.stderr.strip()[-500:]}"
+            )
+
+    @staticmethod
+    def _home(trace: Trace) -> str:
+        return f"/tmp/vf-hermes/{trace.id}"

verifiers/v1/harnesses/hermes_agent/program.py

@@ -0,0 +1,10 @@
+# /// script
+# requires-python = ">=3.11,<3.14"
+# dependencies = ["hermes-agent[acp,mcp]=={version}"]
+# ///
+"""Start Hermes Agent's native ACP server."""
+
+from acp_adapter.entry import main
+
+if __name__ == "__main__":
+    main()

verifiers/v1/task.py

@@ -3,8 +3,8 @@
 `TaskData` is the wire half: a frozen pydantic model carrying the data which
 initializes a task instance. Rides on `trace.task.data` in `traces.jsonl`.
 
-`Task` is the behavior half: runtime prep (`setup`/`finalize`), tool declarations
-(`tools`), and scoring (`@reward`/`@metric`) methods.
+`Task` is the behavior half: runtime prep (`setup`/`finalize`), tool servers
+(`toolsets`), and scoring (`@reward`/`@metric`) methods.
 """
 
 from __future__ import annotations

{verifiers-0.2.2.dev57.dist-info → verifiers-0.2.2.dev59.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev57
+Version: 0.2.2.dev59
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev57.dist-info → verifiers-0.2.2.dev59.dist-info}/RECORD

@@ -171,7 +171,7 @@ verifiers/v1/__init__.py,sha256=Im5qTZy9mH6PRTUi-1V6W7EWKZzJyTWLGKMYALbWz_w,7467
 verifiers/v1/agent.py,sha256=t5Z_iI6Zyt0ieMDI7pfHUVYIKViuSXF-CI_rIApx3f4,32071
 verifiers/v1/artifacts.py,sha256=5eKOjDSUWwj7GrhxILMHdz2ngEONWMdcTGlBcuGgrLU,5834
 verifiers/v1/decorators.py,sha256=9CcCwgZlRjVGPqT5G1CFJUHP4OIMuBGkUJZkcI6dnbk,3865
-verifiers/v1/env.py,sha256=wJp9i49_LKVzMFXkwtt_SaRMUJ84i5r1t3QfNvnt4Ks,18178
+verifiers/v1/env.py,sha256=TGCq_rPp_VdyGAKpHuscSDyIxIGnim2N-dpKkG-J5p0,18175
 verifiers/v1/episode.py,sha256=YtuRTV_PpwDglXFeN6UNg8IswpP1HseFLCYtascGMNI,3218
 verifiers/v1/errors.py,sha256=slZnrtqMo_BgXQV46wJJhV6vvAAnVsuCbVGp2vBVWfs,6888
 verifiers/v1/graph.py,sha256=aBAzh1Ibp3klKNYjvzNyJSbjV8Oi5TJ791dN7o6ex1E,29070
@@ -185,7 +185,7 @@ verifiers/v1/rollout.py,sha256=ehPoN6EnrIg_lvF1-pdf9pS8j47K3z6jDHc66MixzX0,20470
 verifiers/v1/scoring.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/session.py,sha256=J6ZPuvsg6vMTJAs9dMUMtfdjCuvp6Hom877DVdKFKlY,6912
 verifiers/v1/state.py,sha256=pcpN2V6tX7rIO5XdtdUYay1j0ktEwrMtq7w1ApfsPdE,675
-verifiers/v1/task.py,sha256=skqOc-j2AxA6Wz2ikDD03o3cu1cVvr1G6gRJZrpOgfY,9059
+verifiers/v1/task.py,sha256=osuZNjGeNj5NC9TYxxzRQUPnt8D5aNw1OuS45eVWugI,9057
 verifiers/v1/taskset.py,sha256=s9T7v3REU1EKVCu2zORMA3gWinRK6LJk4GUvoKhAmEQ,4337
 verifiers/v1/trace.py,sha256=_bGHOrE6fRLGm0JYc0PrrrO3lyr-qqOF5Y1bHfViwR0,19347
 verifiers/v1/types.py,sha256=1PxamJspmoTc3OlFZAwH6o_2i_Tg9ZNvHxttIHWIxq8,8948
@@ -252,7 +252,7 @@ verifiers/v1/gepa/config.py,sha256=gFn8Re4GoeHJs2PZ-iw4jCEZ72AiM1lNgwaA0uvJlYw,4
 verifiers/v1/gepa/dataset.py,sha256=N2QYqUljHVuAdCbUuydEIlarvAWa6LFdGKu9qsOvjro,2186
 verifiers/v1/gepa/reflection.py,sha256=ptHhx0lcDLcWhWhXuslh9T8gPVNpnX5XLPtXoneW1yA,1054
 verifiers/v1/gepa/runner.py,sha256=-LZEt0lod6NKfyxg2dvOXZjfzTv3AWwendexd1De26c,6260
-verifiers/v1/harnesses/__init__.py,sha256=706ES37vrh_kCbctnbgkXIb1jBtO2ozZY5klBSg8ss0,1460
+verifiers/v1/harnesses/__init__.py,sha256=CDKLCWYqRbVy3NvEzavC12_DaxuoHQfw9AQ8hmlQuBQ,1624
 verifiers/v1/harnesses/bash/__init__.py,sha256=IAV4eMMqDHSl-9ANb0Q3kxmBZ22SeHZFBM448X4gQio,140
 verifiers/v1/harnesses/bash/harness.py,sha256=ZtjhsANuK3vXkf9mRCi8-sBMyPebJMBNHa58vTd2Fm8,5284
 verifiers/v1/harnesses/bash/program.py,sha256=hGv3k9jwoabL045AGiZWiX76Cca3Juk58AaMYeJTEGo,15334
@@ -263,6 +263,9 @@ verifiers/v1/harnesses/claude_code/__init__.py,sha256=JkZQylMSA3o1fvBx_NM41y_umu
 verifiers/v1/harnesses/claude_code/harness.py,sha256=98k5T8d8nFQmxIU-ytkrkrrP6UNR5NxoGhVJRLVa-fI,4015
 verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
 verifiers/v1/harnesses/codex/harness.py,sha256=OG1riQdVmZzVpHENsd0UzsIbdIAxDMHgRikmo73HL5M,13071
+verifiers/v1/harnesses/hermes_agent/__init__.py,sha256=k84Y8UTTf1KOw5LGqPb2dk-hYFagPevDJBOF7qsR_vo,176
+verifiers/v1/harnesses/hermes_agent/harness.py,sha256=AveZi9g4pga9pYy4VAmn_KMKlJw3KhwG45JZcPxbdR4,5094
+verifiers/v1/harnesses/hermes_agent/program.py,sha256=9qPLbUnhox2bGu5ypEEnBhw9QKtYwNvrbl7W_zeNMdk,229
 verifiers/v1/harnesses/kimi_code/__init__.py,sha256=1BEotSfuzmVcE7UrxHeAREM5d7J20cGkonHuowlrzu4,161
 verifiers/v1/harnesses/kimi_code/harness.py,sha256=Pc2uiBsia-uQFuXvaLMFdblE6TS-p0E2cvB6ofRrHK0,4515
 verifiers/v1/harnesses/mini_swe_agent/__init__.py,sha256=1OBu2Y-XqL72-dGhFdgB6sr5uX_peEEXT9vEjPy1ovU,182
@@ -333,8 +336,8 @@ verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/sampling.py,sha256=52TJ5-Hmpvp4YSgKNR-LmAPVwm4RDcH-o2-BnbVTmYQ,965
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev57.dist-info/METADATA,sha256=4FbkVpDnb3GnGhrNC-YaPVwF8qfnOQgOMoHLmWkC57E,4545
-verifiers-0.2.2.dev57.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev57.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev57.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev57.dist-info/RECORD,,
+verifiers-0.2.2.dev59.dist-info/METADATA,sha256=_zlzZC6PmnCHuYIWYn0fKecFkRUb2_ts5uzFM3yM_iQ,4545
+verifiers-0.2.2.dev59.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev59.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev59.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev59.dist-info/RECORD,,