verifiers 0.3.1.dev23__tar.gz → 0.3.1.dev25__tar.gz

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev23
+Version: 0.3.1.dev25
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/verifiers/v1/__init__.py

@@ -81,6 +81,7 @@ from verifiers.v1.trace import (
     Error,
     EvalRunInfo,
     ModelCall,
+    PolicyEvent,
     Reward,
     RunInfo,
     TimeSpan,
@@ -204,6 +205,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "RunInfo",
     "EvalRunInfo",
     "ModelCall",
+    "PolicyEvent",
     "TrainRunInfo",
     "VersionInfo",
     "State",

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/verifiers/v1/agent.py

@@ -19,13 +19,13 @@ from verifiers.v1.clients import (
     ModelContext,
 )
 from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
+from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects import parse_message
 from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
 from verifiers.v1.rollout import Rollout, RolloutTimeouts
 from verifiers.v1.runtimes import (
-    NetworkPolicyConfig,
     Runtime,
     RuntimeConfig,
     SubprocessConfig,

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/verifiers/v1/clients/client.py

@@ -33,14 +33,14 @@ class Client(ABC):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
+        sampling: SamplingConfig,
         session_id: str | None = None,
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> Response:
-        """Run one completion -> a vf `Response`. The eval client forwards the native JSON
-        and parses a copy via `dialect`; the train client renders `body` to token ids.
+        """Run one completion -> a vf `Response`. `body` is the final effective native
+        request after overrides and policy mediation: the eval client forwards it unchanged,
+        while the train client renders it to token ids using the resolved `sampling` config.
         `session_id` is the rollout's trace id (sent as `SESSION_ID_HEADER`); `turn` is the
         graph-resolved prompt prefix, used by train clients for renderer bridging."""
 
@@ -48,14 +48,11 @@ class Client(ABC):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
         session_id: str | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> RelayReply:
-        """Stream a (possibly SSE) response back, relaying the provider's bytes — the proxy's
-        path for a streaming request. Only the relay (eval) client supports it; the renderer
-        generates and cannot stream."""
+        """Stream a response for the final effective `body`, relaying the provider's bytes.
+        Only the relay (eval) client supports it; the renderer generates and cannot stream."""
         raise NotImplementedError(f"{type(self).__name__} does not support streaming")
 
     async def relay_aux(

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/verifiers/v1/clients/eval.py

@@ -70,15 +70,14 @@ class EvalClient(Client):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
+        sampling: SamplingConfig,
         session_id: str | None = None,
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> Response:
         resp = await self._request(
             join_url(self.base_url, dialect.upstream_path),
-            dialect.apply_overrides(body, model, sampling_args),
+            body,
             self._headers(dialect, headers, session_id),
         )
         # A corrupted response (e.g. an HTML error page or a truncated body on a
@@ -164,8 +163,6 @@ class EvalClient(Client):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
         session_id: str | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> RelayReply:
@@ -173,7 +170,7 @@ class EvalClient(Client):
         # between them. Error responses are mapped before any event is handed back.
         resp = await self._request(
             join_url(self.base_url, dialect.upstream_path),
-            dialect.apply_overrides(body, model, sampling_args),
+            body,
             self._headers(dialect, headers, session_id),
             stream=True,
         )

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/verifiers/v1/clients/train.py

@@ -319,8 +319,7 @@ class TrainClient(Client):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
+        sampling: SamplingConfig,
         session_id: str | None = None,
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
@@ -353,7 +352,8 @@ class TrainClient(Client):
         prompt_ids: list[int] | None = None
         multi_modal_data = None
         prompt_attribution: RenderedTokens | None = None
-        raw_sampling = sampling_args.model_dump(exclude_none=True)
+        model = body["model"]
+        raw_sampling = sampling.model_dump(exclude_none=True)
         sampling_params: dict[str, Any] = dict(
             raw_sampling.pop("extra_body", None) or {}
         )

verifiers-0.3.1.dev25/verifiers/v1/configs/runtime.py

@@ -0,0 +1,66 @@
+"""Shared configuration for execution-time network policy."""
+
+from fnmatch import fnmatchcase
+from typing import Self
+from urllib.parse import urlsplit
+
+from pydantic import Field, model_validator
+from pydantic_config import BaseConfig
+
+
+def network_rule_matches(rule: str, scheme: str, host: str, port: int) -> bool:
+    """Match a network-policy host pattern or URL origin. Paths are ignored."""
+    value = rule.lower().rstrip("/")
+    try:
+        parsed = urlsplit(value if "://" in value else f"//{value}")
+        rule_port = parsed.port
+    except ValueError:
+        return False
+    pattern = (parsed.hostname or "").rstrip(".")
+    if not pattern or (parsed.scheme and parsed.scheme != scheme):
+        return False
+    if parsed.scheme and rule_port is None:
+        rule_port = 443 if parsed.scheme == "https" else 80
+    if rule_port is not None and rule_port != port:
+        return False
+    host = host.lower().rstrip(".")
+    return fnmatchcase(host, pattern) or (
+        pattern.startswith("*.") and host == pattern[2:]
+    )
+
+
+class NetworkPolicyConfig(BaseConfig):
+    """Shared execution-time policy surface for runtimes that support it."""
+
+    allow: list[str] = Field(default_factory=lambda: ["*"])
+    """Destinations allowed during execution; `*` is unrestricted and `[]` is
+    framework-only."""
+    block: list[str] = Field(default_factory=list)
+    """Destinations denied during execution; any `*` makes the policy framework-only."""
+
+    @model_validator(mode="after")
+    def validate_network_policy(self) -> Self:
+        if not self.allow or "*" in self.block:
+            # Empty allowlists and wildcard blocks both mean framework-only access.
+            self.allow = []
+            self.block = ["*"]
+        elif self.allow != ["*"] and self.block:
+            raise ValueError(
+                "non-empty concrete allow and block egress lists are mutually exclusive"
+            )
+        return self
+
+    @property
+    def network_restricted(self) -> bool:
+        return "*" not in self.allow or bool(self.block)
+
+    def with_task_network_policy(self, allow: list[str], block: list[str]) -> Self:
+        values = self.model_dump()
+        if "*" in allow:
+            allow = self.allow
+        elif "*" not in self.allow:
+            # Rules are opaque: guessing overlap between different globs could widen the policy.
+            runtime_allow = set(self.allow)
+            allow = [rule for rule in allow if rule in runtime_allow]
+        block = list(dict.fromkeys([*block, *self.block]))
+        return type(self).model_validate({**values, "allow": allow, "block": block})

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/verifiers/v1/dialects/anthropic.py

@@ -7,13 +7,22 @@ trace. `count_tokens` is relayed as native JSON (an `aux_route`), never recorded
 """
 
 import json
+import re
 from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field
+from typing import cast
 
 from anthropic.types import Message as AnthropicMessage
+from anthropic.types import MessageCreateParams
 from anthropic.types import Usage as AnthropicUsage
 
-from verifiers.v1.dialects.base import Dialect, StreamParser, parse_sse_event
+from verifiers.v1.dialects.base import (
+    Dialect,
+    StreamParser,
+    append_user_notice,
+    blocked_url,
+    parse_sse_event,
+)
 from verifiers.v1.types import (
     AssistantMessage,
     ContentPart,
@@ -42,6 +51,54 @@ STOP_REASONS = {
 }
 # Claude may reorder mixed thinking block types between a response and its replay.
 THINKING = ("redacted_thinking", "thinking")
+# These versioned tool families return calls to the harness; every other typed tool may execute
+# at the provider. Anchoring the pattern keeps new versions client-side without treating an
+# arbitrary dated provider tool as safe.
+_CLIENT_TOOL_TYPE = re.compile(r"(?:bash|text_editor|computer|memory)_\d{8}").fullmatch
+_CONTENT_WRAPPERS = (
+    "tool_result",
+    "code_execution_tool_result",
+    "bash_code_execution_tool_result",
+    "text_editor_code_execution_tool_result",
+    "web_search_tool_result",
+    "web_fetch_tool_result",
+    "tool_search_tool_result",
+    "mcp_tool_result",
+    "advisor_tool_result",
+    "code_execution_result",
+    "bash_code_execution_result",
+    "encrypted_code_execution_result",
+    "web_fetch_result",
+)
+_SAFE_CONTENT_TYPES = (
+    "text",
+    "image",
+    "document",
+    "tool_reference",
+    "thinking",
+    "redacted_thinking",
+    "tool_use",
+    "search_result",
+    "server_tool_use",
+    "mid_conv_system",
+    "compaction",
+    "fallback",
+    "mcp_tool_use",
+    "web_search_result",
+    "web_search_tool_result_error",
+    "web_fetch_tool_result_error",
+    "tool_search_tool_search_result",
+    "tool_search_tool_result_error",
+    "code_execution_tool_result_error",
+    "bash_code_execution_tool_result_error",
+    "text_editor_code_execution_tool_result_error",
+    "text_editor_code_execution_create_result",
+    "text_editor_code_execution_str_replace_result",
+    "text_editor_code_execution_view_result",
+    "advisor_result",
+    "advisor_redacted_result",
+    "advisor_tool_result_error",
+)
 
 
 def parse_content(content) -> str | list[ContentPart]:
@@ -68,6 +125,79 @@ def parse_content(content) -> str | list[ContentPart]:
     return parts
 
 
+def blocked_content_path(value, path: str) -> str | None:
+    if isinstance(value, list):
+        for index, item in enumerate(value):
+            if blocked := blocked_content_path(item, f"{path}[{index}]"):
+                return blocked
+        return None
+    if not isinstance(value, dict):
+        return None
+
+    kind = value.get("type")
+    caller = value.get("caller")
+    if caller is not None and not (
+        isinstance(caller, dict) and caller.get("type") == "direct"
+    ):
+        return f"{path}.caller.type"
+    if kind in ("image", "document"):
+        source_path = f"{path}.source"
+        source = value.get("source") or {}
+        if not isinstance(source, dict):
+            return source_path
+        source_kind = source.get("type")
+        if source_kind == "content":
+            return blocked_content_path(source.get("content"), f"{source_path}.content")
+        if source_kind == "url":
+            url = source.get("url")
+            if not isinstance(url, str) or blocked_url(url):
+                return f"{source_path}.url"
+        if source_kind == "file":
+            return (
+                f"{source_path}.file_id"
+                if source.get("file_id")
+                else f"{source_path}.type"
+            )
+        if source_kind not in ("base64", "text", "url"):
+            return f"{source_path}.type"
+
+    if kind in (
+        "container_upload",
+        "code_execution_output",
+        "bash_code_execution_output",
+    ) and value.get("file_id"):
+        return f"{path}.file_id"
+    if kind in _CONTENT_WRAPPERS:
+        return blocked_content_path(value.get("content"), f"{path}.content")
+    return None if kind in _SAFE_CONTENT_TYPES else f"{path}.type"
+
+
+def mediate_content(value, path: str):
+    if not isinstance(value, list):
+        blocked = blocked_content_path(value, path)
+        return ("", [blocked]) if blocked else (value, [])
+
+    mediated = []
+    capabilities = []
+    for index, block in enumerate(value):
+        item_path = f"{path}[{index}]"
+        if isinstance(block, dict) and block.get("type") in _CONTENT_WRAPPERS:
+            if blocked := blocked_content_path({**block, "content": []}, item_path):
+                capabilities.append(blocked)
+                continue
+            content, removed = mediate_content(
+                block.get("content"), f"{item_path}.content"
+            )
+            if removed:
+                block["content"] = content or ""
+                capabilities.extend(removed)
+        elif blocked := blocked_content_path(block, item_path):
+            capabilities.append(blocked)
+            continue
+        mediated.append(block)
+    return mediated, capabilities
+
+
 def parse_messages(body: dict) -> Messages:
     """The request's top-level `system` + `messages` -> typed messages. Assistant turns fold
     their blocks into one message (thinking -> reasoning, tool_use -> tool calls); a user turn's
@@ -265,7 +395,7 @@ class ModdedAnthropicMessage(AnthropicMessage):
     usage: ModdedUsage  # type: ignore[assignment]
 
 
-class AnthropicDialect(Dialect[dict, AnthropicMessage]):
+class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
     sampling_fields = frozenset(
         {
             "temperature",
@@ -283,6 +413,73 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
     upstream_path = "/v1/messages"
     response_type = ModdedAnthropicMessage
 
+    def mediate_external_capabilities(
+        self, body: MessageCreateParams
+    ) -> tuple[MessageCreateParams, list[str]]:
+        mediated = body
+        capabilities: list[str] = []
+
+        for key in ("container", "mcp_servers"):
+            if mediated.pop(key, None):
+                capabilities.append(key)
+
+        system, removed = mediate_content(mediated.get("system"), "system")
+        capabilities.extend(removed)
+        if removed:
+            if system:
+                mediated["system"] = system
+            else:
+                mediated.pop("system")
+
+        for message_index, message in enumerate(mediated.get("messages") or []):
+            if not isinstance(message, dict):
+                continue
+            content, removed = mediate_content(
+                message.get("content"), f"messages[{message_index}].content"
+            )
+            capabilities.extend(removed)
+            if removed:
+                message["content"] = content or ""
+
+        raw_tools = mediated.get("tools")
+        tool_items = raw_tools if isinstance(raw_tools, list) else []
+        if raw_tools is not None and not isinstance(raw_tools, list):
+            capabilities.append("tools")
+        tools = []
+        for index, tool in enumerate(tool_items):
+            kind = tool.get("type") if isinstance(tool, dict) else None
+            if isinstance(tool, dict) and (
+                (isinstance(kind, str) and _CLIENT_TOOL_TYPE(kind))
+                or (kind in (None, "custom") and "input_schema" in tool)
+            ):
+                tools.append(tool)
+                continue
+            capabilities.append(f"tools[{index}].type")
+        if "tools" in mediated:
+            mediated["tools"] = tools
+
+        choice = mediated.get("tool_choice")
+        valid_choice = choice is None
+        if isinstance(choice, dict):
+            kind = choice.get("type")
+            valid_choice = (
+                kind == "none"
+                or bool(tools)
+                and (
+                    kind in ("auto", "any")
+                    or kind == "tool"
+                    and any(tool.get("name") == choice.get("name") for tool in tools)
+                )
+            )
+        if not valid_choice:
+            capabilities.append(
+                "tool_choice.type" if isinstance(choice, dict) else "tool_choice"
+            )
+            mediated.pop("tool_choice", None)
+
+        append_user_notice(mediated.setdefault("messages", []))
+        return mediated, capabilities
+
     def auth_headers(self, api_key: str) -> dict[str, str]:
         return {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
 
@@ -296,7 +493,9 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
             "error": {"type": "invalid_request_error", "message": message},
         }
 
-    def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]:
+    def parse_request(
+        self, body: MessageCreateParams
+    ) -> tuple[Messages, list[Tool] | None]:
         tools = [
             Tool(
                 name=t["name"],
@@ -314,7 +513,7 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
     def stream_parser(self) -> StreamParser:
         return AnthropicStreamParser(self.validate_response)
 
-    def parse_sampling(self, body: dict) -> Sampling:
+    def parse_sampling(self, body: MessageCreateParams) -> Sampling:
         settings = {k: v for k, v in body.items() if k in self.sampling_fields}
         # Lift `output_config.effort` (where `apply_overrides` puts the eval's
         # reasoning effort) onto the typed knob; keep any other output-config keys.
@@ -328,7 +527,9 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
                 settings.pop("output_config")
         return Sampling.model_validate(settings)
 
-    def apply_overrides(self, body: dict, model: str, sampling: SamplingConfig) -> dict:
+    def apply_overrides(
+        self, body: MessageCreateParams, model: str, sampling: SamplingConfig
+    ) -> MessageCreateParams:
         # Preserve native fields except the eval's model + sampling. `temperature`/`top_p` are
         # authoritative (always dropped, the eval's applied if set); `max_tokens` is required by
         # the API, so the program's is kept unless the eval sets one.
@@ -350,4 +551,4 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
             for k, v in body.items()
             if k not in ("temperature", "top_p") and k not in overrides
         }
-        return {**steered, **overrides}
+        return cast(MessageCreateParams, {**steered, **overrides})

{verifiers-0.3.1.dev23 → verifiers-0.3.1.dev25}/verifiers/v1/dialects/base.py

@@ -27,6 +27,45 @@ RespT = TypeVar("RespT", bound=BaseModel)
 
 logger = logging.getLogger(__name__)
 
+PROVIDER_CAPABILITY_POLICY_CODE = "provider_capability_unavailable"
+CAPABILITY_NOTICE = (
+    "Network protocol blocked fetching a resource. Continue without those capabilities; "
+    "use local tools or inline data already present in the conversation, and do not retry "
+    "the blocked provider-side operation."
+)
+
+
+def blocked_url(value: str) -> bool:
+    """Whether a provider-resolved resource is not inline data."""
+    return not value.lower().startswith("data:")
+
+
+def append_user_notice(
+    messages: list,
+    *,
+    text_type: str = "text",
+    message_type: str | None = None,
+) -> None:
+    """Add stable restricted-network context to the earliest user input."""
+    part = {"type": text_type, "text": CAPABILITY_NOTICE}
+    for message in messages:
+        if not isinstance(message, dict) or message.get("role") != "user":
+            continue
+        content = message.get("content")
+        if isinstance(content, list):
+            message["content"] = [*content, part]
+        elif isinstance(content, str):
+            message["content"] = (
+                f"{content}\n\n{CAPABILITY_NOTICE}" if content else CAPABILITY_NOTICE
+            )
+        else:
+            message["content"] = [part]
+        return
+    message = {"role": "user", "content": [part]}
+    if message_type is not None:
+        message["type"] = message_type
+    messages.append(message)
+
 
 def is_sse_done_event(raw: bytes) -> bool:
     """Whether one complete SSE event carries the DONE sentinel."""
@@ -147,6 +186,12 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         """An error payload in this format's error shape (OpenAI by default)."""
         return {"error": {"message": message, "type": "invalid_request_error"}}
 
+    @abstractmethod
+    def mediate_external_capabilities(self, body: ReqT) -> tuple[ReqT, list[str]]:
+        """Remove provider-side capabilities during restricted execution. Implementations add
+        the same policy context on every call because the agent does not retain injected request
+        content. Returned paths never contain request values."""
+
     @abstractmethod
     def parse_request(self, body: ReqT) -> tuple[Messages, list[Tool] | None]:
         """The native request -> vf prompt + tools (for the trace)."""
@@ -174,5 +219,5 @@ class Dialect(ABC, Generic[ReqT, RespT]):
     @abstractmethod
     def apply_overrides(self, body: ReqT, model: str, sampling: SamplingConfig) -> ReqT:
         """Return `body` with the eval's `model` + `sampling` imposed in this protocol's shape —
-        the only field mutation the proxy makes to the native JSON object. Model overlays;
-        sampling is authoritative (the program's sampling keys are dropped, the eval's applied)."""
+        model overlays; sampling is authoritative (the program's sampling keys are dropped, the
+        eval's applied). Capability mediation may subsequently remove restricted fields."""