verifiers 0.3.2.dev125__tar.gz → 0.3.2.dev127__tar.gz

{verifiers-0.3.2.dev125 → verifiers-0.3.2.dev127}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev125
+Version: 0.3.2.dev127
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev125 → verifiers-0.3.2.dev127}/verifiers/v1/acp/__init__.py

@@ -85,6 +85,13 @@ class ACPHarness(Harness[ConfigT]):
             )
             trace.add_semantic_edges(edge_set)
 
+    async def gate_tools(
+        self, config: ACPConfig, runtime: Runtime, url: str, secret: str
+    ) -> None:
+        """Configure the agent to ask before every tool call, so each request reaches the
+        runner's gate (`/tool` at `url`, keyed by `secret`). A harness that advertises
+        `SUPPORTS_TOOL_INTERCEPTION` implements this with the agent's own config."""
+
     @abstractmethod
     async def prepare_acp(
         self,
@@ -116,6 +123,8 @@ class ACPHarness(Harness[ConfigT]):
         config = await self.prepare_acp(
             ctx, trace, runtime, endpoint, secret, mcp_urls, data
         )
+        if tool_interception_url is not None:
+            await self.gate_tools(config, runtime, tool_interception_url, secret)
         return ACPHarnessSession(
             self,
             ctx,
@@ -268,6 +277,12 @@ class ACPHarnessSession(HarnessSession):
             "system_prompt": self.config.system_prompt or "",
             "session_meta": self.config.session_meta or {},
             "client_capabilities": self.config.client_capabilities or {},
+            "tool_interception": {
+                "url": self.tool_interception_url,
+                "secret": self.secret,
+            }
+            if self.tool_interception_url
+            else None,
         }
         async with self._lock:
             if self._closed:

{verifiers-0.3.2.dev125 → verifiers-0.3.2.dev127}/verifiers/v1/acp/runner.py

@@ -1,6 +1,6 @@
 # /// script
 # requires-python = ">=3.10,<3.15"
-# dependencies = ["agent-client-protocol==0.12.1"]
+# dependencies = ["agent-client-protocol==0.12.1", "httpx"]
 # ///
 """Run harness segments through an ACP agent."""
 
@@ -14,6 +14,7 @@ from contextlib import AsyncExitStack, suppress
 from dataclasses import asdict, dataclass
 from typing import Any
 
+import httpx
 from acp import (
     PROTOCOL_VERSION,
     Client,
@@ -45,8 +46,42 @@ class ACPTurn:
     update_metadata: list[dict[str, Any]]
 
 
+class ToolGate:
+    """The rollout's `/tool` gate, asked before every tool call the agent wants to run."""
+
+    def __init__(self, url: str, secret: str) -> None:
+        self.url = url
+        self.client = httpx.AsyncClient(
+            headers={"Authorization": f"Bearer {secret}"},
+            timeout=httpx.Timeout(120, connect=5),
+        )
+
+    async def decision(self, tool_call_id: str, arguments: Any) -> str:
+        try:
+            # pi-acp wraps extension confirmations in a separate UI permission call.
+            if (
+                tool_call_id.startswith("pi-ui-")
+                and isinstance(arguments, dict)
+                and arguments.get("method") == "confirm"
+            ):
+                tool_call_id, arguments = (
+                    arguments["title"],
+                    json.loads(arguments["message"]),
+                )
+            response = await self.client.post(
+                self.url, json={"tool_call_id": tool_call_id, "arguments": arguments}
+            )
+            response.raise_for_status()
+            return response.json()["action"]
+        except Exception as error:  # noqa: BLE001 - an unreachable gate lets nothing run
+            print(f"tool gate denied {tool_call_id}: {error}", file=sys.stderr)
+            return "deny"
+
+
 class VerifiersACPClient(Client):
     def __init__(self) -> None:
+        self.gate: ToolGate | None = None
+        self.prompt_task: asyncio.Task | None = None
         self.visible_reply = ""
         self.message_id: str | None = None
         self.stop_reason: str | None = None
@@ -90,9 +125,23 @@ class VerifiersACPClient(Client):
         options: list[PermissionOption],
         **kwargs: Any,
     ) -> RequestPermissionResponse:
+        """Ask the rollout before execution: deny rejects one call; stop cancels the turn."""
+        kinds = ("allow_once", "allow_always")
+        if self.gate is not None:
+            decision = await self.gate.decision(
+                tool_call.tool_call_id, tool_call.raw_input
+            )
+            if decision == "stop":
+                self.stop_reason = "cancelled"
+                if self.prompt_task is not None:
+                    self.prompt_task.cancel()
+                return RequestPermissionResponse(
+                    outcome=DeniedOutcome(outcome="cancelled")
+                )
+            if decision != "allow":
+                kinds = ("reject_once", "reject_always")
         option = next(
-            (item for item in options if item.kind in ("allow_once", "allow_always")),
-            None,
+            (item for kind in kinds for item in options if item.kind == kind), None
         )
         outcome = (
             AllowedOutcome(outcome="selected", option_id=option.option_id)
@@ -165,12 +214,21 @@ async def prompt(
     if not blocks:
         raise ValueError("ACP prompt has no content")
     try:
-        response = await connection.prompt(session_id=session_id, prompt=blocks)
+        client.prompt_task = asyncio.create_task(
+            connection.prompt(session_id=session_id, prompt=blocks)
+        )
+        response = await client.prompt_task
         client.stop_reason = response.stop_reason
         client.response_metadata = dict(response.field_meta or {})
+    except asyncio.CancelledError:
+        if client.stop_reason != "cancelled":
+            raise
+        await connection.cancel(session_id=session_id)
     except RequestError as error:
         detail = error.data.get("details") if isinstance(error.data, dict) else None
         raise RuntimeError(detail or str(error)) from error
+    finally:
+        client.prompt_task = None
     return client.turn_result()
 
 
@@ -222,6 +280,9 @@ class ACPSession:
         self.is_new = True
 
     async def run(self, config: dict) -> ACPTurn:
+        gate = config.get("tool_interception")
+        if gate and self.client.gate is None:
+            self.client.gate = ToolGate(gate["url"], gate["secret"])
         if self.connection is None:
             await self.start(config)
         assert self.session_id is not None
@@ -253,6 +314,9 @@ class ACPSession:
             try:
                 await self.stack.aclose()
             finally:
+                if self.client.gate is not None:
+                    await self.client.gate.client.aclose()
+                    self.client.gate = None
                 self._reset()
         return response_metadata
 

verifiers-0.3.2.dev127/verifiers/v1/configs/runtime.py

@@ -0,0 +1,126 @@
+"""Shared configuration for execution-time network policy."""
+
+from fnmatch import fnmatchcase
+from glob import has_magic
+from itertools import product
+from typing import Self
+from urllib.parse import SplitResult, urlsplit
+
+from pydantic import Field, model_validator
+from pydantic_config import BaseConfig
+
+
+def parse_network_rule(rule: str) -> tuple[SplitResult, str, int | None]:
+    """Normalize host/port while retaining URL syntax for backend validation."""
+    parsed = urlsplit(rule if "://" in rule else f"//{rule}")
+    port = parsed.port
+    if parsed.scheme and port is None:
+        port = 443 if parsed.scheme == "https" else 80
+    return parsed, (parsed.hostname or "").lower().rstrip("."), port
+
+
+def network_rule_matches(rule: str, scheme: str, host: str, port: int) -> bool:
+    """Match a network-policy host pattern or URL origin. Paths are ignored."""
+    try:
+        parsed, pattern, rule_port = parse_network_rule(rule)
+    except ValueError:
+        return False
+    if not pattern or (parsed.scheme and parsed.scheme != scheme):
+        return False
+    if rule_port is not None and rule_port != port:
+        return False
+    host = host.lower().rstrip(".")
+    return fnmatchcase(host, pattern) or (
+        pattern.startswith("*.") and host == pattern[2:]
+    )
+
+
+def intersect_network_hosts(left: str, right: str) -> str:
+    """Return the narrower host pattern, rejecting unprovable glob overlaps."""
+    for parent, child in ((left, right), (right, left)):
+        if parent in ("*", child):
+            return child
+        # Matching a pattern as text proves containment only for domain suffixes.
+        if has_magic(child) and not (
+            parent.startswith("*.") and not has_magic(parent[2:])
+        ):
+            continue
+        if fnmatchcase(child, parent) or child == parent.removeprefix("*."):
+            return child
+    if all(has_magic(host) for host in (left, right)) and any(
+        has_magic(host.removeprefix("*.")) for host in (left, right)
+    ):
+        raise ValueError(f"cannot intersect network hosts {left!r} and {right!r}")
+    return ""
+
+
+class NetworkPolicyConfig(BaseConfig):
+    """Shared execution-time policy surface for runtimes that support it."""
+
+    allow: list[str] = Field(default_factory=lambda: ["*"])
+    """Destinations allowed during execution; `*` is unrestricted and `[]` is
+    framework-only."""
+    block: list[str] = Field(default_factory=list)
+    """Destinations denied during execution; any `*` makes the policy framework-only."""
+
+    @model_validator(mode="after")
+    def validate_network_policy(self) -> Self:
+        if not self.allow or "*" in self.block:
+            # Empty allowlists and wildcard blocks both mean framework-only access.
+            self.allow = []
+            self.block = ["*"]
+        elif self.allow != ["*"] and self.block:
+            raise ValueError(
+                "non-empty concrete allow and block egress lists are mutually exclusive"
+            )
+        return self
+
+    @property
+    def network_restricted(self) -> bool:
+        return "*" not in self.allow or bool(self.block)
+
+    def permits(self, scheme: str, host: str, port: int) -> bool:
+        """Whether the destination is allowed by the configured egress rules."""
+        return not any(
+            network_rule_matches(rule, scheme, host, port) for rule in self.block
+        ) and any(network_rule_matches(rule, scheme, host, port) for rule in self.allow)
+
+    def with_task_network_policy(self, allow: list[str], block: list[str]) -> Self:
+        values = self.model_dump()
+        # Intersection must not erase syntax the runtime cannot enforce.
+        task = type(self).model_validate({**values, "allow": allow, "block": block})
+        allow, block = task.allow, task.block
+        if "*" in allow:
+            allow = self.allow
+        elif "*" not in self.allow:
+            intersection = []
+            for left, right in product(allow, self.allow):
+                if left == right:
+                    intersection.append(left)
+                    continue
+                a, a_host, a_port = parse_network_rule(left)
+                b, b_host, b_port = parse_network_rule(right)
+                # Bare paths can be provider-specific CIDRs, not URL paths to discard.
+                if any(
+                    not url.hostname or (url.path and not url.scheme) for url in (a, b)
+                ):
+                    raise ValueError(
+                        f"cannot intersect network rules {left!r} and {right!r}"
+                    )
+                if (
+                    len({a.scheme, b.scheme} - {""}) > 1
+                    or len({a_port, b_port} - {None}) > 1
+                ):
+                    continue
+                host = intersect_network_hosts(a_host, b_host)
+                if not host:
+                    continue
+                scheme = a.scheme or b.scheme
+                port = a_port if a_port is not None else b_port
+                authority = f"[{host}]" if ":" in host else host
+                if port is not None:
+                    authority = f"{authority}:{port}"
+                intersection.append(f"{scheme}://{authority}" if scheme else authority)
+            allow = list(dict.fromkeys(intersection))
+        block = list(dict.fromkeys([*block, *self.block]))
+        return type(self).model_validate({**values, "allow": allow, "block": block})

{verifiers-0.3.2.dev125 → verifiers-0.3.2.dev127}/verifiers/v1/dialects/anthropic.py

@@ -18,9 +18,9 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects.base import (
     Dialect,
     RawRequest,
+    RequestFilter,
     StreamParser,
     append_user_notice,
-    blocked_url,
     parse_sse_event,
     provider_domains,
 )
@@ -129,89 +129,66 @@ def parse_content(content) -> str | list[ContentPart]:
     return parts
 
 
-def blocked_content_path(
-    value, path: str, policy: NetworkPolicyConfig, blocked_urls: list[str]
-) -> str | None:
-    if isinstance(value, list):
-        for index, item in enumerate(value):
-            if blocked := blocked_content_path(
-                item, f"{path}[{index}]", policy, blocked_urls
-            ):
-                return blocked
-        return None
-    if not isinstance(value, dict):
-        return None
+class AnthropicRequestFilter(RequestFilter):
+    wrappers = _CONTENT_WRAPPERS
 
-    kind = value.get("type")
-    caller = value.get("caller")
-    if caller is not None and not (
-        isinstance(caller, dict) and caller.get("type") == "direct"
-    ):
-        return f"{path}.caller.type"
-    if kind in ("image", "document"):
-        source_path = f"{path}.source"
-        source = value.get("source") or {}
-        if not isinstance(source, dict):
-            return source_path
-        source_kind = source.get("type")
-        if source_kind == "content":
-            return blocked_content_path(
-                source.get("content"), f"{source_path}.content", policy, blocked_urls
+    def tool(self, tool, path: str) -> dict | None:
+        kind = tool.get("type") if isinstance(tool, dict) else None
+        if isinstance(kind, str) and _WEB_TOOL_TYPE(kind):
+            callers = tool.get("allowed_callers")
+            filter_key = "blocked_domains" if self.policy.block else "allowed_domains"
+            other_key = "allowed_domains" if self.policy.block else "blocked_domains"
+            domains = (
+                provider_domains(self.policy, tool.get(filter_key))
+                # Anthropic does not support combining allow and block filters.
+                if tool.get(other_key) in (None, [])
+                and (
+                    callers is None or isinstance(callers, list) and "direct" in callers
+                )
+                else []
             )
-        if source_kind == "url":
-            url = source.get("url")
-            if not isinstance(url, str) or blocked_url(url, policy, blocked_urls):
+            if domains:
+                web_tool = {**tool, filter_key: domains, "allowed_callers": ["direct"]}
+                web_tool.pop(other_key, None)
+                return web_tool
+        if isinstance(tool, dict) and (
+            (isinstance(kind, str) and _CLIENT_TOOL_TYPE(kind))
+            or (kind in (None, "custom") and "input_schema" in tool)
+        ):
+            return tool
+        self.capabilities.append(f"{path}.type")
+        return None
+
+    def blocked_part(self, value: dict, path: str) -> str | None:
+        kind = value.get("type")
+        if kind in ("image", "document"):
+            source_path = f"{path}.source"
+            source = value.get("source") or {}
+            if not isinstance(source, dict):
+                return source_path
+            source_kind = source.get("type")
+            if source_kind == "content":
+                return self.blocked(source.get("content"), f"{source_path}.content")
+            if source_kind == "url" and self.blocked_url(source.get("url")):
                 return f"{source_path}.url"
-        if source_kind == "file":
-            return (
-                f"{source_path}.file_id"
-                if source.get("file_id")
-                else f"{source_path}.type"
-            )
-        if source_kind not in ("base64", "text", "url"):
-            return f"{source_path}.type"
-
-    if kind in (
-        "container_upload",
-        "code_execution_output",
-        "bash_code_execution_output",
-    ) and value.get("file_id"):
-        return f"{path}.file_id"
-    if kind in _CONTENT_WRAPPERS:
-        return blocked_content_path(
-            value.get("content"), f"{path}.content", policy, blocked_urls
-        )
-    return None if kind in _SAFE_CONTENT_TYPES else f"{path}.type"
-
-
-def mediate_content(
-    value, path: str, policy: NetworkPolicyConfig, blocked_urls: list[str]
-):
-    if not isinstance(value, list):
-        blocked = blocked_content_path(value, path, policy, blocked_urls)
-        return ("", [blocked]) if blocked else (value, [])
-
-    mediated = []
-    capabilities = []
-    for index, block in enumerate(value):
-        item_path = f"{path}[{index}]"
-        if isinstance(block, dict) and block.get("type") in _CONTENT_WRAPPERS:
-            if blocked := blocked_content_path(
-                {**block, "content": []}, item_path, policy, blocked_urls
-            ):
-                capabilities.append(blocked)
-                continue
-            content, removed = mediate_content(
-                block.get("content"), f"{item_path}.content", policy, blocked_urls
-            )
-            if removed:
-                block["content"] = content or ""
-                capabilities.extend(removed)
-        elif blocked := blocked_content_path(block, item_path, policy, blocked_urls):
-            capabilities.append(blocked)
-            continue
-        mediated.append(block)
-    return mediated, capabilities
+            if source_kind == "file":
+                return (
+                    f"{source_path}.file_id"
+                    if source.get("file_id")
+                    else f"{source_path}.type"
+                )
+            if source_kind not in ("base64", "text", "url"):
+                return f"{source_path}.type"
+
+        if kind in (
+            "container_upload",
+            "code_execution_output",
+            "bash_code_execution_output",
+        ) and value.get("file_id"):
+            return f"{path}.file_id"
+        if kind in self.wrappers:
+            return self.blocked(value.get("content"), f"{path}.content")
+        return None if kind in _SAFE_CONTENT_TYPES else f"{path}.type"
 
 
 def content_to_wire(content) -> str | list[dict]:
@@ -464,80 +441,29 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
         self, body: RawRequest, policy: NetworkPolicyConfig
     ) -> tuple[RawRequest, list[str]]:
         mediated = body
-        capabilities: list[str] = []
-        blocked_urls: list[str] = []
+        request_filter = AnthropicRequestFilter(policy)
+        capabilities = request_filter.capabilities
 
         for key in ("container", "mcp_servers"):
             if mediated.pop(key, None):
                 capabilities.append(key)
 
-        system, removed = mediate_content(
-            mediated.get("system"), "system", policy, blocked_urls
-        )
-        capabilities.extend(removed)
-        if removed:
-            if system:
-                mediated["system"] = system
-            else:
-                mediated.pop("system")
+        if (
+            request_filter.content(mediated, "system", "system")
+            and not mediated["system"]
+        ):
+            mediated.pop("system")
 
         for message_index, message in enumerate(mediated.get("messages") or []):
             if not isinstance(message, dict):
                 continue
-            content, removed = mediate_content(
-                message.get("content"),
+            request_filter.content(
+                message,
+                "content",
                 f"messages[{message_index}].content",
-                policy,
-                blocked_urls,
             )
-            capabilities.extend(removed)
-            if removed:
-                message["content"] = content or ""
-
-        raw_tools = mediated.get("tools")
-        tool_items = raw_tools if isinstance(raw_tools, list) else []
-        if raw_tools is not None and not isinstance(raw_tools, list):
-            capabilities.append("tools")
-        tools = []
-        for index, tool in enumerate(tool_items):
-            kind = tool.get("type") if isinstance(tool, dict) else None
-            if (
-                isinstance(tool, dict)
-                and isinstance(kind, str)
-                and _WEB_TOOL_TYPE(kind)
-            ):
-                callers = tool.get("allowed_callers")
-                filter_key = "blocked_domains" if policy.block else "allowed_domains"
-                other_key = "allowed_domains" if policy.block else "blocked_domains"
-                domains = (
-                    provider_domains(policy, tool.get(filter_key))
-                    # Anthropic does not support combining allow and block filters.
-                    if tool.get(other_key) in (None, [])
-                    and (
-                        callers is None
-                        or isinstance(callers, list)
-                        and "direct" in callers
-                    )
-                    else []
-                )
-                if domains:
-                    web_tool = {
-                        **tool,
-                        filter_key: domains,
-                        "allowed_callers": ["direct"],
-                    }
-                    web_tool.pop(other_key, None)
-                    tools.append(web_tool)
-                    continue
-                capabilities.append(f"tools[{index}].type")
-                continue
-            if isinstance(tool, dict) and (
-                (isinstance(kind, str) and _CLIENT_TOOL_TYPE(kind))
-                or (kind in (None, "custom") and "input_schema" in tool)
-            ):
-                tools.append(tool)
-                continue
-            capabilities.append(f"tools[{index}].type")
+
+        tools = request_filter.tools(mediated.get("tools"))
         if "tools" in mediated:
             mediated["tools"] = tools
 
@@ -562,7 +488,8 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
 
         if capabilities:
             append_user_notice(
-                mediated.setdefault("messages", []), blocked_urls=blocked_urls
+                mediated.setdefault("messages", []),
+                blocked_urls=request_filter.blocked_urls,
             )
         return mediated, capabilities