verifiers 0.4.1.dev25__tar.gz → 0.4.1.dev27__tar.gz

{verifiers-0.4.1.dev25 → verifiers-0.4.1.dev27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev25
+Version: 0.4.1.dev27
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev25 → verifiers-0.4.1.dev27}/tests/integration/conftest.py

@@ -67,6 +67,9 @@ def pytest_configure(config) -> None:
     inherits `PYTHONPATH` but not pytest's in-process `pythonpath`. Put the fixture dir on
     `PYTHONPATH` so a fixture server module (e.g. `tool_response_image_v1`)
     resolves there too — an installed example package (e.g. `glossary`) already would."""
+    config.addinivalue_line(
+        "markers", "opencode: integration cells on the OpenCode harness"
+    )
     fixtures = str(Path(__file__).parent / "fixtures")
     existing = os.environ.get("PYTHONPATH", "")
     if fixtures not in existing.split(os.pathsep):

{verifiers-0.4.1.dev25 → verifiers-0.4.1.dev27}/tests/integration/grid.py

@@ -52,6 +52,7 @@ HARNESSES: list[str | dict] = [
     "kimi-code",
     "pi",
     "openclaw",
+    "opencode",
     "mini-swe-agent",
     "prime-agent",
 ]

{verifiers-0.4.1.dev25 → verifiers-0.4.1.dev27}/verifiers/v1/harnesses/__init__.py

@@ -19,6 +19,7 @@ from verifiers.v1.harnesses.mini_swe_agent import (
 )
 from verifiers.v1.harnesses.null import NullHarness, NullHarnessConfig
 from verifiers.v1.harnesses.openclaw import OpenClawHarness, OpenClawHarnessConfig
+from verifiers.v1.harnesses.opencode import OpenCodeHarness, OpenCodeHarnessConfig
 from verifiers.v1.harnesses.pi import PiHarness, PiHarnessConfig
 from verifiers.v1.harnesses.prime_agent import (
     PrimeAgentHarness,
@@ -46,6 +47,8 @@ __all__ = [
     "NullHarnessConfig",
     "OpenClawHarness",
     "OpenClawHarnessConfig",
+    "OpenCodeHarness",
+    "OpenCodeHarnessConfig",
     "PiHarness",
     "PiHarnessConfig",
     "PrimeAgentHarness",

verifiers-0.4.1.dev27/verifiers/v1/harnesses/opencode/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.harnesses.opencode.harness import (
+    OpenCodeHarness,
+    OpenCodeHarnessConfig,
+)
+
+__all__ = ["OpenCodeHarness", "OpenCodeHarnessConfig"]

verifiers-0.4.1.dev27/verifiers/v1/harnesses/opencode/harness.py

@@ -0,0 +1,141 @@
+"""Run OpenCode 2's native ACP server against interception."""
+
+import json
+from pathlib import Path
+from typing import Literal
+
+from verifiers.v1.acp import ACPConfig, ACPHarness
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
+from verifiers.v1.harnesses.node import ensure_node
+from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
+from verifiers.v1.runtimes import Runtime
+from verifiers.v1.task import TaskData
+from verifiers.v1.trace import Trace
+
+OPENCODE_DIR = "/var/tmp/vf-opencode"
+STATE_ROOT = "/tmp/vf-opencode-runs"
+KEY_VAR = "VF_OPENCODE_API_KEY"
+MCP_READY = (Path(__file__).resolve().parent / "mcp-ready.js").read_bytes()
+
+INSTALL = r"""
+set -e
+export PATH="/var/tmp/vf-node/bin:$PATH"
+npm install --prefix "$VF_OPENCODE_DIR" --no-audit --no-fund --omit=dev \
+    "@opencode/cli@$VF_OPENCODE_VERSION" >/dev/null
+"""
+
+
+class OpenCodeHarnessConfig(HarnessConfig):
+    version: PinnedVersion = "2.0.26"
+    """OpenCode 2 release to install, pinned for reproducibility."""
+    transport: Literal["chat_completions", "responses", "anthropic_messages"] = (
+        "chat_completions"
+    )
+    """Model API transport."""
+    disabled_tools: list[str] | None = None
+    """OpenCode permission actions to deny, such as shell, read, or edit."""
+
+
+class OpenCodeHarness(ACPHarness[OpenCodeHarnessConfig]):
+    APPENDS_SYSTEM_PROMPT = True
+    SUPPORTS_MCP = True
+    SUPPORTS_SKILLS = True
+
+    async def setup(self, runtime: Runtime) -> None:
+        await ensure_node(runtime)
+        directory = f"{OPENCODE_DIR}/{self.config.version}"
+        await ensure_installed(
+            runtime,
+            directory=directory,
+            ready=f"[ -x {directory}/node_modules/.bin/opencode ]",
+            install=INSTALL,
+            env={
+                **self.config.resolved_env,
+                "VF_OPENCODE_DIR": directory,
+                "VF_OPENCODE_VERSION": self.config.version,
+            },
+            label="OpenCode",
+        )
+        await super().setup(runtime)
+
+    async def prepare_acp(
+        self,
+        ctx: ModelContext,
+        trace: Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: TaskData,
+    ) -> ACPConfig:
+        root = f"{STATE_ROOT}/{trace.id}"
+        config_dir = f"{root}/config/opencode"
+        await self.install_skills(runtime, f"{config_dir}/skills")
+        system_prompt, prompt = self.resolve_prompt(data)
+        if system_prompt:
+            await runtime.write(f"{config_dir}/AGENTS.md", system_prompt.encode())
+
+        package = {
+            "chat_completions": "openai-compatible",
+            "responses": "openai-compatible/responses",
+            "anthropic_messages": "anthropic-compatible",
+        }[self.config.transport]
+        settings = {
+            "model": "intercept/model",
+            "providers": {
+                "intercept": {
+                    "env": [KEY_VAR],
+                    "package": f"@opencode/ai/providers/{package}",
+                    "settings": {"baseURL": endpoint},
+                    "models": {"model": {"modelID": ctx.model}},
+                }
+            },
+            "default_agent": "build",
+            # Auxiliary title/summary model calls must not become rollout turns.
+            "agents": {"title": {"disabled": True}, "summary": {"disabled": True}},
+            "permissions": [
+                {"action": "*", "resource": "*", "effect": "allow"},
+                *(
+                    {"action": tool, "resource": "*", "effect": "deny"}
+                    for tool in self.config.disabled_tools or []
+                ),
+            ],
+            "update": "disable",
+            "share": "disabled",
+            "snapshots": False,
+        }
+        if mcp_urls:
+            plugin_dir = f"{root}/mcp-ready"
+            await runtime.write(f"{plugin_dir}/index.js", MCP_READY)
+            settings["plugins"] = [
+                {
+                    "package": plugin_dir,
+                    "options": {
+                        "servers": list(mcp_urls),
+                        "timeout": int(self.config.tool_timeout * 1000),
+                    },
+                }
+            ]
+        return ACPConfig(
+            command=[
+                f"{OPENCODE_DIR}/{self.config.version}/node_modules/.bin/opencode",
+                "acp",
+            ],
+            env={
+                **self.config.resolved_env,
+                KEY_VAR: secret,
+                "XDG_CONFIG_HOME": f"{root}/config",
+                "XDG_DATA_HOME": f"{root}/data",
+                "XDG_CACHE_HOME": f"{root}/cache",
+                "XDG_STATE_HOME": f"{root}/state",
+                "OPENCODE_CONFIG_DIR": config_dir,
+                "OPENCODE_CONFIG_CONTENT": json.dumps(settings),
+                "OPENCODE_DISABLE_AUTOUPDATE": "1",
+                "OPENCODE_DISABLE_MODELS_FETCH": "1",
+            },
+            prompt=prompt,
+        )
+
+    async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
+        await remove_dir(runtime, f"{STATE_ROOT}/{trace.id}", "OpenCode state")

verifiers-0.4.1.dev27/verifiers/v1/harnesses/opencode/mcp-ready.js

@@ -0,0 +1,44 @@
+import { Context, Effect } from "effect";
+
+export default {
+  id: "verifiers.mcp-ready",
+  effect: Effect.fn(function* (ctx) {
+    // ACP registrations bypass OpenCode's configured MCP server timeout defaults.
+    yield* ctx.mcp.transform((editor) => {
+      for (const name of ctx.options.servers) {
+        editor.update(name, (server) => {
+          server.timeout = { ...server.timeout, execution: ctx.options.timeout };
+        });
+      }
+    });
+    yield* ctx.session.hook(
+      "prompt",
+      Effect.fn(function* () {
+        // OpenCode 2.0.26's public plugin API omits discovered MCP tools. Read
+        // its native service in the prompt's location context, including empty servers.
+        const mcp = yield* Context.Service()("@opencode/MCP");
+        const servers = yield* mcp.servers();
+        for (const name of ctx.options.servers) {
+          const server = servers.find((server) => server.name === name);
+          if (server?.status.status !== "connected") {
+            return yield* Effect.die(
+              new Error(`MCP server ${name} is not connected: ${JSON.stringify(server?.status)}`),
+            );
+          }
+        }
+        const expected = (yield* mcp.tools())
+          .filter((tool) => ctx.options.servers.includes(tool.server))
+          .map((tool) => `${tool.server}_${tool.name}`.replace(/[^a-zA-Z0-9_-]/g, "_"));
+        // ACP session/new awaits connection, but OpenCode refreshes its tool
+        // registry asynchronously. Do not let the prompt snapshot an old catalog.
+        yield* Effect.gen(function* () {
+          while (true) {
+            const registered = new Set((yield* ctx.tool.list()).map((tool) => tool.id));
+            if (expected.every((name) => registered.has(name))) return;
+            yield* Effect.sleep("10 millis");
+          }
+        }).pipe(Effect.timeout(ctx.options.timeout), Effect.orDie);
+      }),
+    );
+  }),
+};

{verifiers-0.4.1.dev25 → verifiers-0.4.1.dev27}/verifiers/v1/rollout.py

@@ -648,10 +648,61 @@ class Rollout:
             }
             logger.warning("harness scoring failed", exc_info=True)
 
+    async def _close_harness(self) -> None:
+        """Close the harness session, then the tool servers and interception. Idempotent.
+
+        Generation has already completed, so a teardown failure is logged rather than
+        raised: it must not discard an otherwise scoreable (or already scored)
+        trajectory."""
+        if self._harness_session is not None:
+            try:
+                await self._harness_session.close()
+            except Exception:
+                logger.warning(
+                    "harness session close failed (rollout %s)",
+                    self.trace.id,
+                    exc_info=True,
+                )
+        try:
+            await self._stack.aclose()
+        except Exception:
+            logger.warning(
+                "tool server or interception teardown failed (rollout %s)",
+                self.trace.id,
+                exc_info=True,
+            )
+
+    def _agent_finished(self) -> bool:
+        """Whether the agent ended its last segment on its own, or waits between the
+        segments of an exchange: nothing it runs is still acting on the box."""
+        return not self._failed and self.trace.stop_condition in (
+            None,
+            "agent_completed",
+        )
+
+    def _metrics_deadline(self) -> float:
+        """Best-effort metrics get at most 30 seconds, and a tenth of a finite scoring budget."""
+        budget = (
+            min(30.0, self._timeouts.scoring / 10)
+            if self._timeouts.scoring is not None
+            else 30.0
+        )
+        return asyncio.get_running_loop().time() + budget
+
     async def close(self) -> Trace:
-        """Finish the rollout: tool servers and interception down, task `finalize`
-        and per-rollout scoring (skipped when the run already failed — but a stopped
-        run is complete and scores its partial trajectory), then runtime teardown.
+        """Finish the rollout: task `finalize` and per-rollout scoring (skipped when the
+        run already failed, but a stopped run is complete and scores its partial
+        trajectory), the harness session, tool servers and interception, then runtime
+        teardown.
+
+        An agent that finished its turn keeps its harness session open while its
+        environment is collected and, when grading runs in the agent's own box, graded:
+        harnesses stop the processes their commands started when the session closes,
+        and a service the task asked the agent to leave running is part of what gets
+        graded. A run stopped mid-turn (a timeout or a stop condition) closes the
+        session first, so an agent that may still be acting cannot change the box while
+        it is graded. Isolated grading runs in its own box after the session closes.
+        Harness metrics run after the close, which hands over the session's final state.
         Idempotent; always returns the trace."""
         if self._closed:
             return self.trace
@@ -659,67 +710,29 @@ class Rollout:
         trace = self.trace
         runtime = self.runtime
         try:
-            if self._harness_session is not None:
-                try:
-                    await self._harness_session.close()
-                except Exception:
-                    # Generation already completed. A transport teardown failure
-                    # must not discard its otherwise scoreable trajectory.
-                    logger.warning(
-                        "harness session close failed (rollout %s)",
-                        trace.id,
-                        exc_info=True,
-                    )
-            try:
-                await self._stack.aclose()
-            finally:
-                if trace.timing.agent.start and not trace.timing.agent.end:
-                    trace.timing.agent.end = time.time()
-                trace.notify()
+            if trace.timing.agent.start and not trace.timing.agent.end:
+                trace.timing.agent.end = time.time()
+            if not self._agent_finished():
+                await self._close_harness()
+            trace.notify()
             if not self._failed and self._opened:
                 assert runtime is not None
                 assert self.deployment is not None
                 grader = self._grader
                 isolated = self._scoring_policy.mode == "isolated"
+                loop = asyncio.get_running_loop()
                 trace.timing.finalize.start = time.time()
+                finalize_deadline = (
+                    loop.time() + self._timeouts.finalize
+                    if self._timeouts.finalize is not None
+                    else None
+                )
                 scoring_elapsed = 0.0
-                early_metrics = (
-                    isolated and grader is not None
-                ) or self._on_score is not None
-                if early_metrics:
-                    scoring_started = asyncio.get_running_loop().time()
-                    # Metrics are best-effort: reserve at least 90% of a finite
-                    # scoring budget for verification, with a 30-second ceiling.
-                    # Capture solver metrics before stopping its workspace or letting
-                    # an agentic judge change files and execution state in place.
-                    try:
-                        async with (
-                            asyncio.timeout(
-                                min(30.0, self._timeouts.scoring / 10)
-                                if self._timeouts.scoring is not None
-                                else 30.0
-                            ),
-                            boundary(HarnessError, "harness scoring"),
-                        ):
-                            await self.harness.score(
-                                trace,
-                                self._agent_runtime or runtime,
-                            )
-                    except Exception as error:
-                        # Metrics are best-effort; their failure must not prevent
-                        # collecting the submission and running its fresh verifier.
-                        trace.info["harness_scoring_error"] = {
-                            "type": type(error).__name__,
-                            "message": str(error),
-                        }
-                        logger.warning("harness scoring failed", exc_info=True)
-                    scoring_elapsed = (
-                        asyncio.get_running_loop().time() - scoring_started
-                    )
+                metrics_done = False
                 finalize_phase = "task finalization"
                 try:
                     async with (
-                        asyncio.timeout(self._timeouts.finalize),
+                        asyncio.timeout_at(finalize_deadline),
                         boundary(TaskError, "task finalize"),
                     ):
                         await invoke(
@@ -739,8 +752,24 @@ class Rollout:
                                 self.task.data.artifacts,
                                 max_bytes=self.task.data.artifact_max_bytes,
                             )
-                        if grader is not None:
-                            finalize_phase = "service finalization"
+                    if grader is not None:
+                        # The submission is collected: close the solver's session, then
+                        # capture its metrics before the workspace stops and the verifier
+                        # starts. Neither counts against finalization; the metrics count
+                        # against scoring, as they did when they ran first.
+                        interlude_started = loop.time()
+                        await self._close_harness()
+                        metrics_started = loop.time()
+                        await self._score_harness(runtime, self._metrics_deadline())
+                        metrics_done = True
+                        scoring_elapsed = loop.time() - metrics_started
+                        if finalize_deadline is not None:
+                            finalize_deadline += loop.time() - interlude_started
+                        finalize_phase = "service finalization"
+                        async with (
+                            asyncio.timeout_at(finalize_deadline),
+                            boundary(TaskError, "task finalize"),
+                        ):
                             await self._cleanup_harness(runtime)
                             await self.deployment.stop_execution()
                             await self.task.finalize_services(
@@ -753,15 +782,9 @@ class Rollout:
                 except TimeoutError:
                     self.timeout("finalize")
                     if isolated:
-                        if not early_metrics:
-                            # Deferred scoring has not captured metrics yet.
-                            deadline = (
-                                asyncio.get_running_loop().time()
-                                + self._timeouts.scoring
-                                if self._timeouts.scoring is not None
-                                else None
-                            )
-                            await self._score_harness(runtime, deadline)
+                        await self._close_harness()
+                        if not metrics_done:
+                            await self._score_harness(runtime, self._metrics_deadline())
                         raise TaskError(
                             f"{finalize_phase} exceeded its finalize deadline"
                         ) from None
@@ -770,26 +793,12 @@ class Rollout:
                 trace.timing.scoring.start = now
                 trace.notify()
                 scoring_deadline = (
-                    asyncio.get_running_loop().time()
-                    + max(0.0, self._timeouts.scoring - scoring_elapsed)
+                    loop.time() + max(0.0, self._timeouts.scoring - scoring_elapsed)
                     if self._timeouts.scoring is not None
                     else None
                 )
-                if not early_metrics:
-                    # Collect metrics before a shared verifier can exhaust the budget.
-                    metrics_budget = (
-                        min(30.0, self._timeouts.scoring / 10)
-                        if self._timeouts.scoring is not None
-                        else 30.0
-                    )
-                    await self._score_harness(
-                        runtime, asyncio.get_running_loop().time() + metrics_budget
-                    )
                 try:
-                    if (
-                        scoring_deadline is not None
-                        and asyncio.get_running_loop().time() >= scoring_deadline
-                    ):
+                    if scoring_deadline is not None and loop.time() >= scoring_deadline:
                         raise TimeoutError("scoring budget exhausted")
                     async with (
                         asyncio.timeout_at(
@@ -847,6 +856,12 @@ class Rollout:
                         self.fail(
                             TaskError("verification exceeded its scoring deadline")
                         )
+                finally:
+                    if grader is None:
+                        # Graded in the agent's own box: now the session can close, and
+                        # the metrics read what it handed over and left behind.
+                        await self._close_harness()
+                        await self._score_harness(runtime, self._metrics_deadline())
                 trace.timing.scoring.end = time.time()
                 trace.notify()
         except Exception as e:  # noqa: BLE001 - finalize boundary records every rollout failure