verifiers 0.4.1.dev29__tar.gz → 0.4.1.dev31__tar.gz

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev29
+Version: 0.4.1.dev31
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/pyproject.toml

@@ -231,6 +231,7 @@ markers = [
     "modal: integration cells touching the modal runtime (external sandbox; local-only, excluded in CI)",
     "e2b: integration cells touching the e2b runtime (external sandbox; local-only, excluded in CI)",
     "colocated: integration cells with a tool server colocated in the harness runtime",
+    "service: integration cells with a tool server in a Compose service beside the harness",
     "null: integration cells on the null harness",
     "bash: integration cells on the bash harness",
     "claude_code: integration cells on the claude-code harness",

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/tests/integration/conftest.py

@@ -46,9 +46,11 @@ def harness_runtime(request) -> dict:
 @pytest.fixture
 def tool_runtime(request) -> dict:
     """A `taskset.task.tools` override placing the tool server: `colocated` (inside the harness's
-    runtime) or its own runtime, by type."""
+    runtime), `service` (in the task's `tools` Compose service) or its own runtime, by type."""
     if request.param == "colocated":
         return {"colocated": True}
+    if request.param == "service":
+        return {"service": "tools"}
     return {"runtime": {"type": request.param}}
 
 

verifiers-0.4.1.dev31/tests/integration/fixtures/echo_tool_compose/docker-compose.yaml

@@ -0,0 +1,7 @@
+# echo-tool-v1 with its tool server placed in a Compose service: the server runs in
+# `tools`, beside the workspace (`main`, the runtime's image), and the harness reaches it
+# over the Compose network by the service's name.
+services:
+  tools:
+    image: python:3.11-slim
+    command: [sleep, infinity]

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/tests/integration/fixtures/echo_tool_v1.py

@@ -2,19 +2,23 @@
 
 The v1 tool fixture for the integration grid. The task constructs an `EchoToolset`
 (`vf.Toolset`) in `Task.toolsets`, with one `@vf.tool` method whose placement is
-CLI-tunable (`--taskset.task.tools.colocated`, `--taskset.task.tools.runtime.type`):
-it runs colocated in the harness's runtime or in its own runtime, and the harness
-must reach it wherever it lives. The tool stamps its output with a token the prompt
+CLI-tunable (`--taskset.task.tools.colocated`, `--taskset.task.tools.service`,
+`--taskset.task.tools.runtime.type`): it runs colocated in the harness's runtime, in the
+task's `tools` Compose service (echo_tool_compose), or in its own runtime, and the
+harness must reach it wherever it lives. The tool stamps its output with a token the prompt
 never reveals, so the reward is 1.0 only if the model actually called the tool —
 trivial when the infra works, impossible when it doesn't. The tool is task-agnostic,
 so it would also serve taskset-scoped (`Taskset.toolsets`).
 """
 
+from pathlib import Path
+
 import verifiers.v1 as vf
 from verifiers.v1.types import content_text
 
 PHRASE = "hello world"
 ECHO_TOKEN = "ok-7f3"  # the tool stamps this; only a real tool call can surface it
+COMPOSE = Path(__file__).with_name("echo_tool_compose")
 
 
 class EchoToolset(vf.Toolset[vf.ToolsetConfig]):
@@ -35,6 +39,11 @@ class EchoToolTask(vf.Task[vf.TaskData, vf.State, EchoToolTaskConfig]):
     def toolsets(cls, config: EchoToolTaskConfig) -> list[vf.Toolset]:
         return [EchoToolset(config.tools)]
 
+    def deployment_spec(self) -> vf.ComposeSpec | None:
+        if self.config.tools.service is None:
+            return None
+        return vf.ComposeSpec(directory=str(COMPOSE), file="docker-compose.yaml")
+
     @vf.reward(weight=1.0)
     async def echoed(self, trace: vf.Trace) -> float:
         # A stamped TOOL result proves the tool really ran with the phrase.

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/tests/integration/grid.py

@@ -59,6 +59,8 @@ HARNESSES: list[str | dict] = [
 RUNTIMES = ["subprocess", "docker", "podman", "apptainer", "prime", "modal", "e2b"]
 ENGINES = ("podman", "apptainer")
 HARNESS_RUNTIMES = [runtime for runtime in RUNTIMES if runtime not in ENGINES]
+# Runtimes whose VMs run a task's Compose services.
+COMPOSE_VMS = ("prime", "modal")
 
 # Harnesses that cannot run against the CI model yet.
 # TODO: drop each entry once its fix lands.
@@ -208,7 +210,7 @@ def placements(
     tool_runtimes: tuple[str, ...] = ("colocated", *RUNTIMES),
 ) -> list:
     """`(harness_runtime, tool_runtime)` cells for a tool server placed colocated in the
-    harness's box or in its own runtime."""
+    harness's box, in a Compose service beside it (`service`), or in its own runtime."""
     cells = []
     for harness_runtime in harness_runtimes:
         for tool_runtime in tool_runtimes:
@@ -220,8 +222,15 @@ def placements(
             ]
             if missing:
                 marks.append(pytest.mark.skip(reason=f"needs {missing[0]} on the host"))
+            elif tool_runtime == "service" and harness_runtime not in COMPOSE_VMS:
+                marks.append(
+                    pytest.mark.skip(
+                        reason="a tool server in a Compose service needs Compose on a "
+                        "Prime or Modal VM"
+                    )
+                )
             elif (
-                tool_runtime != "colocated"
+                tool_runtime not in ("colocated", "service")
                 and not runtime_cls(tool_runtime).supports_expose
             ):
                 marks.append(

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/tests/integration/test_multi_turn_tool.py

@@ -1,11 +1,13 @@
 """One task, one MCP tool: the model calls a tool whose output carries a stamp the prompt
 never reveals (echo-tool-v1), so reward 1.0 proves the tool ran. Every MCP harness reaches
 it colocated across the harness x runtime grid; the chat loop reaches it across its
-placements (colocated in the harness's box, or its own runtime) on the runtime x runtime
-grid."""
+placements (colocated in the harness's box, in a Compose service beside it, or its own
+runtime) on the runtime x runtime grid."""
 
 import pytest
-from grid import grid, placements
+from grid import RUNTIMES, grid, placements
+
+import verifiers.v1 as vf
 
 
 @pytest.mark.parametrize("harness,harness_runtime", grid(needs=("mcp",)), indirect=True)
@@ -29,11 +31,16 @@ async def test_tool(run, harness, harness_runtime):
     assert trace.reward == 1.0
 
 
-@pytest.mark.parametrize("harness_runtime,tool_runtime", placements(), indirect=True)
+@pytest.mark.parametrize(
+    "harness_runtime,tool_runtime",
+    placements(tool_runtimes=("colocated", "service", *RUNTIMES)),
+    indirect=True,
+)
 async def test_tool_placement(run, harness_runtime, tool_runtime):
     """A `vf.Toolset` across its placement (`tool_runtime`: colocated in the harness's
-    runtime, or its own runtime) x the harness `runtime`, on the chat loop. Eval-wide SHARED servers
-    are a different scope (`Taskset.toolsets`), covered by `test_shared_tool`."""
+    runtime, in a Compose service beside it, or its own runtime) x the harness `runtime`,
+    on the chat loop. Eval-wide SHARED servers are a different scope (`Taskset.toolsets`),
+    covered by `test_shared_tool`."""
     (episode,) = await run(
         "echo-tool-v1",
         harness="null",
@@ -51,3 +58,18 @@ async def test_tool_placement(run, harness_runtime, tool_runtime):
     assert trace.tools
     (echo_tool,) = [tool for tool in trace.tools if tool.name.endswith("back")]
     assert "message" in echo_tool.parameters.get("properties", {})
+
+
+@pytest.mark.parametrize(
+    "placement",
+    [
+        {"colocated": True},
+        {"url": "http://example.com/mcp"},
+        {"runtime": {"type": "prime"}},
+    ],
+)
+def test_service_placement_is_exclusive(placement):
+    """A server in a Compose service runs there: not colocated, remote or in a runtime of
+    its own."""
+    with pytest.raises(ValueError):
+        vf.ToolsetConfig(service="tools", **placement)

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/verifiers/v1/dialects/anthropic.py

@@ -24,6 +24,7 @@ from verifiers.v1.dialects.base import (
     parse_sse_event,
     provider_domains,
 )
+from verifiers.v1.errors import ProviderError
 from verifiers.v1.types import (
     AssistantMessage,
     ContentPart,
@@ -51,6 +52,18 @@ STOP_REASONS: dict[str, FinishReason] = {
     "tool_use": "tool_calls",
     "stop_sequence": "stop",
 }
+ERROR_TYPES = {
+    400: "invalid_request_error",
+    401: "authentication_error",
+    403: "permission_error",
+    404: "not_found_error",
+    413: "request_too_large",
+    429: "rate_limit_error",
+    502: "api_error",
+    529: "overloaded_error",
+}
+ERROR_STATUSES = {kind: status for status, kind in ERROR_TYPES.items()}
+
 # Claude may reorder mixed thinking block types between a response and its replay.
 # Native tool events share the final rank, preserving their relative order.
 THINKING_ORDER = {"redacted_thinking": 0, "thinking": 1}
@@ -356,6 +369,11 @@ class AnthropicStreamParser(StreamParser):
         if event is None:
             return
         kind = event.get("type")
+        if kind == "error":
+            # HTTP 200 streams can fail in-band; preserve the provider's cause and IDs.
+            detail = event.get("error") or {}
+            status = ERROR_STATUSES.get(detail.get("type"), 502)
+            raise ProviderError(json.dumps(event), status_code=status)
         if kind == "message_start":
             self.message = event.get("message") or {}
         elif kind == "content_block_start":
@@ -516,7 +534,14 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
         # Anthropic's own keepalive; its SDKs skip it anywhere in the stream.
         return b'event: ping\ndata: {"type": "ping"}\n\n'
 
-    def stream_error(self, error: dict) -> bytes:
+    def stream_error(self, error: dict, *, status_code: int | None = None) -> bytes:
+        # Once HTTP 200 is committed, clients classify retries from the SSE error type.
+        if status_code is not None:
+            kind = ERROR_TYPES.get(
+                status_code,
+                "api_error" if status_code >= 500 else "invalid_request_error",
+            )
+            error = {**error, "error": {**error.get("error", {}), "type": kind}}
         # The Anthropic SDKs raise only on a named `error` event.
         return b"event: error\ndata: " + json.dumps(error).encode() + b"\n\n"
 

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/verifiers/v1/dialects/base.py

@@ -318,7 +318,7 @@ class Dialect(ABC, Generic[RespT]):
         # Don't terminate an empty event; some SSE clients try to JSON-decode it.
         return b": keepalive\n"
 
-    def stream_error(self, error: dict) -> bytes:
+    def stream_error(self, error: dict, *, status_code: int | None = None) -> bytes:
         """An `error_body` as an SSE event, for a failure after the stream is committed.
         OpenAI SDKs raise on any event carrying `error`."""
         return b"data: " + json.dumps(error).encode() + b"\n\n"

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/verifiers/v1/dialects/responses.py

@@ -687,7 +687,7 @@ class ResponsesDialect(Dialect[OpenAIResponse]):
         payload = {"type": kind, "sequence_number": 0, "response": _KEEPALIVE_RESPONSE}
         return f"event: {kind}\ndata: {json.dumps(payload)}\n\n".encode()
 
-    def stream_error(self, error: dict) -> bytes:
+    def stream_error(self, error: dict, *, status_code: int | None = None) -> bytes:
         # `response.failed` is what Responses clients (codex) act on; the `error` key is what
         # the OpenAI SDK raises on.
         message = error["error"]["message"]

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/verifiers/v1/interception/server.py

@@ -288,7 +288,9 @@ async def _buffered_stream(
                 await stream.write(
                     replay.body
                     if response.status < 400
-                    else dialect.stream_error(from_json(replay.body))
+                    else dialect.stream_error(
+                        from_json(replay.body), status_code=response.status
+                    )
                 )
                 await stream.write_eof()
             except ConnectionResetError:

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/verifiers/v1/mcp/launch.py

@@ -11,7 +11,7 @@ import sys
 import sysconfig
 import tempfile
 import uuid
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from dataclasses import dataclass, field
 from functools import cache
 from importlib.metadata import PathDistribution, distribution, packages_distributions
@@ -325,6 +325,7 @@ async def serve_in_runtime(
     runtime: Runtime,
     *,
     exposed: bool,
+    listen_all: bool = False,
     state_url: str | None = None,
     state_secret: str = "",
 ) -> int:
@@ -345,7 +346,7 @@ async def serve_in_runtime(
         # Keep provider temp files in the runtime workdir so cleanup removes them.
         assert runtime.info.id is not None
         env["TMPDIR"] = runtime.info.id
-    if exposed and runtime.published_port is not None:
+    if listen_all or (exposed and runtime.published_port is not None):
         env["MCP_HOST"] = "0.0.0.0"
     fixed = runtime.published_port if exposed else None
     port_file = None
@@ -432,9 +433,38 @@ async def _serve(
     *,
     state_secret: str = "",
     state_base: str | None = None,
+    services: Mapping[str, Runtime] | None = None,
 ):
     cfg = server.config
     colocated = getattr(cfg, "colocated", False)
+    if (service := getattr(cfg, "service", None)) is not None:
+        # The harness reaches the server over the deployment's Compose network.
+        if services is None or service not in services:
+            raise ToolsetError(
+                f"server {server.server_name!r} runs in Compose service {service!r}, "
+                "which needs a task whose Compose services define it"
+            )
+        runtime = services[service]
+        if runtime.is_local:
+            # A local service cannot reach the host-loopback state channel.
+            raise ToolsetError(
+                f"server {server.server_name!r} runs in Compose service {service!r}, "
+                "which needs Compose on a Prime or Modal VM"
+            )
+        port = await serve_in_runtime(
+            server,
+            runtime,
+            exposed=False,
+            listen_all=True,
+            state_url=(
+                runtime.host_url(f"{state_base.rstrip('/')}/state")
+                if state_base
+                else None
+            ),
+            state_secret=state_secret,
+        )
+        yield _ServedServer(f"http://{service}:{port}/mcp", runtime)
+        return
     async with contextlib.AsyncExitStack() as stack:
         # Colocated servers inherit the harness cut. A separately provisioned filtered
         # server has neither that lifecycle nor a published port after isolation;
@@ -501,6 +531,7 @@ async def serve(
     *,
     state_secret: str = "",
     state_base: str | None = None,
+    services: Mapping[str, Runtime] | None = None,
 ):
     """Serve one MCP server and yield the URL visible to its consumer."""
     async with _serve(
@@ -509,6 +540,7 @@ async def serve(
         harness_is_local,
         state_secret=state_secret,
         state_base=state_base,
+        services=services,
     ) as served:
         yield served.url
 
@@ -616,6 +648,7 @@ async def serve_tools(
     state_secret: str = "",
     state_route: str = "",
     state_base: str | None = None,
+    services: Mapping[str, Runtime] | None = None,
 ):
     """Bring up a rollout's tool servers and yield `{name: url}` the harness reaches: the
     task-scoped `toolsets` are launched by `serve` (placement off each one's `config`; the
@@ -656,6 +689,7 @@ async def serve_tools(
                         harness_runtime,
                         state_secret=state_secret,
                         state_base=state_base,
+                        services=services,
                     )
                 )
                 logger.info("tool server '%s': %s", name, urls[name])

verifiers-0.4.1.dev31/verifiers/v1/mcp/toolset.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from pydantic import model_validator
+from pydantic_config import BaseConfig
+
+from verifiers.v1.mcp.server import ConfigT, ServerBase
+from verifiers.v1.runtimes import RuntimeConfig, SubprocessConfig
+from verifiers.v1.state import StateT
+from verifiers.v1.utils.decorators import discover_decorated
+
+if TYPE_CHECKING:
+    from mcp.server.mcpserver import MCPServer
+
+
+class ToolsetConfig(BaseConfig):
+    colocated: bool = False
+    runtime: RuntimeConfig = SubprocessConfig()
+    url: str | None = None
+    service: str | None = None
+    """Run the server in this service of the task's Compose services (on a Prime or
+    Modal VM). The harness reaches it over the Compose network by the service's name,
+    so the workspace needs no access to whatever the server fronts."""
+
+    @model_validator(mode="after")
+    def _one_placement(self):
+        if self.service is None:
+            return self
+        if self.colocated or self.url:
+            raise ValueError(
+                "a tool server placed in a Compose service is neither colocated nor remote"
+            )
+        # The service is the server's runtime; another one, or its policy, would go unused.
+        if not isinstance(self.runtime, SubprocessConfig):
+            raise ValueError(  # noqa: TRY004 - Pydantic validators must raise ValueError
+                "a tool server placed in a Compose service runs in that service and "
+                f"takes no runtime of its own (got {self.runtime.type})"
+            )
+        return self
+
+
+class SharedToolsetConfig(BaseConfig):
+    runtime: RuntimeConfig = SubprocessConfig()
+    url: str | None = None
+
+
+class Toolset(ServerBase[ConfigT, StateT]):
+    def register(self, mcp: MCPServer) -> None:
+        for fn in discover_decorated(self, "tool"):
+            mcp.add_tool(
+                self._with_state(fn),
+                name=getattr(fn, "tool_name", None) or fn.__name__,
+                description=(fn.__doc__ or "").strip() or None,
+            )

{verifiers-0.4.1.dev29 → verifiers-0.4.1.dev31}/verifiers/v1/rollout.py

@@ -7,6 +7,7 @@ import time
 from collections.abc import Awaitable, Callable
 from contextlib import AsyncExitStack
 from dataclasses import dataclass
+from urllib.parse import urlsplit
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.agent import AgentConfig
@@ -364,6 +365,14 @@ class Rollout:
                 )
                 self._endpoint = runtime.host_url(f"{base_url.rstrip('/')}/v1")
                 self._secret = model_secret
+                services = (
+                    {
+                        name: self.deployment.target(name)
+                        for name in self.deployment.target_backends
+                    }
+                    if self.deployment.spec is not None
+                    else {}
+                )
                 self._urls = await self._stack.enter_async_context(
                     serve_tools(
                         toolsets,
@@ -372,12 +381,18 @@ class Rollout:
                         state_secret=state_secret,
                         state_route=self.trace.id,
                         state_base=base_url,
+                        services=services,
                     )
                 )
                 # Setup and service provisioning are complete. Apply the runtime's
                 # execution policy while preserving the framework routes the agent uses.
+                # Compose service names resolve inside the deployment, not as egress.
                 await self.deployment.prepare_execution(
-                    [self._endpoint, *self._urls.values()]
+                    [
+                        url
+                        for url in [self._endpoint, *self._urls.values()]
+                        if urlsplit(url).hostname not in services
+                    ]
                 )
                 async with (
                     boundary(HarnessError, "opening harness session"),

verifiers-0.4.1.dev29/verifiers/v1/mcp/toolset.py

@@ -1,34 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from pydantic_config import BaseConfig
-
-from verifiers.v1.mcp.server import ConfigT, ServerBase
-from verifiers.v1.runtimes import RuntimeConfig, SubprocessConfig
-from verifiers.v1.state import StateT
-from verifiers.v1.utils.decorators import discover_decorated
-
-if TYPE_CHECKING:
-    from mcp.server.mcpserver import MCPServer
-
-
-class ToolsetConfig(BaseConfig):
-    colocated: bool = False
-    runtime: RuntimeConfig = SubprocessConfig()
-    url: str | None = None
-
-
-class SharedToolsetConfig(BaseConfig):
-    runtime: RuntimeConfig = SubprocessConfig()
-    url: str | None = None
-
-
-class Toolset(ServerBase[ConfigT, StateT]):
-    def register(self, mcp: MCPServer) -> None:
-        for fn in discover_decorated(self, "tool"):
-            mcp.add_tool(
-                self._with_state(fn),
-                name=getattr(fn, "tool_name", None) or fn.__name__,
-                description=(fn.__doc__ or "").strip() or None,
-            )