verifiers 0.3.2.dev126__tar.gz → 0.3.2.dev127__tar.gz

{verifiers-0.3.2.dev126 → verifiers-0.3.2.dev127}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev126
+Version: 0.3.2.dev127
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.3.2.dev127/verifiers/v1/configs/runtime.py

@@ -0,0 +1,126 @@
+"""Shared configuration for execution-time network policy."""
+
+from fnmatch import fnmatchcase
+from glob import has_magic
+from itertools import product
+from typing import Self
+from urllib.parse import SplitResult, urlsplit
+
+from pydantic import Field, model_validator
+from pydantic_config import BaseConfig
+
+
+def parse_network_rule(rule: str) -> tuple[SplitResult, str, int | None]:
+    """Normalize host/port while retaining URL syntax for backend validation."""
+    parsed = urlsplit(rule if "://" in rule else f"//{rule}")
+    port = parsed.port
+    if parsed.scheme and port is None:
+        port = 443 if parsed.scheme == "https" else 80
+    return parsed, (parsed.hostname or "").lower().rstrip("."), port
+
+
+def network_rule_matches(rule: str, scheme: str, host: str, port: int) -> bool:
+    """Match a network-policy host pattern or URL origin. Paths are ignored."""
+    try:
+        parsed, pattern, rule_port = parse_network_rule(rule)
+    except ValueError:
+        return False
+    if not pattern or (parsed.scheme and parsed.scheme != scheme):
+        return False
+    if rule_port is not None and rule_port != port:
+        return False
+    host = host.lower().rstrip(".")
+    return fnmatchcase(host, pattern) or (
+        pattern.startswith("*.") and host == pattern[2:]
+    )
+
+
+def intersect_network_hosts(left: str, right: str) -> str:
+    """Return the narrower host pattern, rejecting unprovable glob overlaps."""
+    for parent, child in ((left, right), (right, left)):
+        if parent in ("*", child):
+            return child
+        # Matching a pattern as text proves containment only for domain suffixes.
+        if has_magic(child) and not (
+            parent.startswith("*.") and not has_magic(parent[2:])
+        ):
+            continue
+        if fnmatchcase(child, parent) or child == parent.removeprefix("*."):
+            return child
+    if all(has_magic(host) for host in (left, right)) and any(
+        has_magic(host.removeprefix("*.")) for host in (left, right)
+    ):
+        raise ValueError(f"cannot intersect network hosts {left!r} and {right!r}")
+    return ""
+
+
+class NetworkPolicyConfig(BaseConfig):
+    """Shared execution-time policy surface for runtimes that support it."""
+
+    allow: list[str] = Field(default_factory=lambda: ["*"])
+    """Destinations allowed during execution; `*` is unrestricted and `[]` is
+    framework-only."""
+    block: list[str] = Field(default_factory=list)
+    """Destinations denied during execution; any `*` makes the policy framework-only."""
+
+    @model_validator(mode="after")
+    def validate_network_policy(self) -> Self:
+        if not self.allow or "*" in self.block:
+            # Empty allowlists and wildcard blocks both mean framework-only access.
+            self.allow = []
+            self.block = ["*"]
+        elif self.allow != ["*"] and self.block:
+            raise ValueError(
+                "non-empty concrete allow and block egress lists are mutually exclusive"
+            )
+        return self
+
+    @property
+    def network_restricted(self) -> bool:
+        return "*" not in self.allow or bool(self.block)
+
+    def permits(self, scheme: str, host: str, port: int) -> bool:
+        """Whether the destination is allowed by the configured egress rules."""
+        return not any(
+            network_rule_matches(rule, scheme, host, port) for rule in self.block
+        ) and any(network_rule_matches(rule, scheme, host, port) for rule in self.allow)
+
+    def with_task_network_policy(self, allow: list[str], block: list[str]) -> Self:
+        values = self.model_dump()
+        # Intersection must not erase syntax the runtime cannot enforce.
+        task = type(self).model_validate({**values, "allow": allow, "block": block})
+        allow, block = task.allow, task.block
+        if "*" in allow:
+            allow = self.allow
+        elif "*" not in self.allow:
+            intersection = []
+            for left, right in product(allow, self.allow):
+                if left == right:
+                    intersection.append(left)
+                    continue
+                a, a_host, a_port = parse_network_rule(left)
+                b, b_host, b_port = parse_network_rule(right)
+                # Bare paths can be provider-specific CIDRs, not URL paths to discard.
+                if any(
+                    not url.hostname or (url.path and not url.scheme) for url in (a, b)
+                ):
+                    raise ValueError(
+                        f"cannot intersect network rules {left!r} and {right!r}"
+                    )
+                if (
+                    len({a.scheme, b.scheme} - {""}) > 1
+                    or len({a_port, b_port} - {None}) > 1
+                ):
+                    continue
+                host = intersect_network_hosts(a_host, b_host)
+                if not host:
+                    continue
+                scheme = a.scheme or b.scheme
+                port = a_port if a_port is not None else b_port
+                authority = f"[{host}]" if ":" in host else host
+                if port is not None:
+                    authority = f"{authority}:{port}"
+                intersection.append(f"{scheme}://{authority}" if scheme else authority)
+            allow = list(dict.fromkeys(intersection))
+        block = list(dict.fromkeys([*block, *self.block]))
+        return type(self).model_validate({**values, "allow": allow, "block": block})

{verifiers-0.3.2.dev126 → verifiers-0.3.2.dev127}/verifiers/v1/dialects/anthropic.py

@@ -18,9 +18,9 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects.base import (
     Dialect,
     RawRequest,
+    RequestFilter,
     StreamParser,
     append_user_notice,
-    blocked_url,
     parse_sse_event,
     provider_domains,
 )
@@ -129,89 +129,66 @@ def parse_content(content) -> str | list[ContentPart]:
     return parts
 
 
-def blocked_content_path(
-    value, path: str, policy: NetworkPolicyConfig, blocked_urls: list[str]
-) -> str | None:
-    if isinstance(value, list):
-        for index, item in enumerate(value):
-            if blocked := blocked_content_path(
-                item, f"{path}[{index}]", policy, blocked_urls
-            ):
-                return blocked
-        return None
-    if not isinstance(value, dict):
-        return None
+class AnthropicRequestFilter(RequestFilter):
+    wrappers = _CONTENT_WRAPPERS
 
-    kind = value.get("type")
-    caller = value.get("caller")
-    if caller is not None and not (
-        isinstance(caller, dict) and caller.get("type") == "direct"
-    ):
-        return f"{path}.caller.type"
-    if kind in ("image", "document"):
-        source_path = f"{path}.source"
-        source = value.get("source") or {}
-        if not isinstance(source, dict):
-            return source_path
-        source_kind = source.get("type")
-        if source_kind == "content":
-            return blocked_content_path(
-                source.get("content"), f"{source_path}.content", policy, blocked_urls
+    def tool(self, tool, path: str) -> dict | None:
+        kind = tool.get("type") if isinstance(tool, dict) else None
+        if isinstance(kind, str) and _WEB_TOOL_TYPE(kind):
+            callers = tool.get("allowed_callers")
+            filter_key = "blocked_domains" if self.policy.block else "allowed_domains"
+            other_key = "allowed_domains" if self.policy.block else "blocked_domains"
+            domains = (
+                provider_domains(self.policy, tool.get(filter_key))
+                # Anthropic does not support combining allow and block filters.
+                if tool.get(other_key) in (None, [])
+                and (
+                    callers is None or isinstance(callers, list) and "direct" in callers
+                )
+                else []
             )
-        if source_kind == "url":
-            url = source.get("url")
-            if not isinstance(url, str) or blocked_url(url, policy, blocked_urls):
+            if domains:
+                web_tool = {**tool, filter_key: domains, "allowed_callers": ["direct"]}
+                web_tool.pop(other_key, None)
+                return web_tool
+        if isinstance(tool, dict) and (
+            (isinstance(kind, str) and _CLIENT_TOOL_TYPE(kind))
+            or (kind in (None, "custom") and "input_schema" in tool)
+        ):
+            return tool
+        self.capabilities.append(f"{path}.type")
+        return None
+
+    def blocked_part(self, value: dict, path: str) -> str | None:
+        kind = value.get("type")
+        if kind in ("image", "document"):
+            source_path = f"{path}.source"
+            source = value.get("source") or {}
+            if not isinstance(source, dict):
+                return source_path
+            source_kind = source.get("type")
+            if source_kind == "content":
+                return self.blocked(source.get("content"), f"{source_path}.content")
+            if source_kind == "url" and self.blocked_url(source.get("url")):
                 return f"{source_path}.url"
-        if source_kind == "file":
-            return (
-                f"{source_path}.file_id"
-                if source.get("file_id")
-                else f"{source_path}.type"
-            )
-        if source_kind not in ("base64", "text", "url"):
-            return f"{source_path}.type"
-
-    if kind in (
-        "container_upload",
-        "code_execution_output",
-        "bash_code_execution_output",
-    ) and value.get("file_id"):
-        return f"{path}.file_id"
-    if kind in _CONTENT_WRAPPERS:
-        return blocked_content_path(
-            value.get("content"), f"{path}.content", policy, blocked_urls
-        )
-    return None if kind in _SAFE_CONTENT_TYPES else f"{path}.type"
-
-
-def mediate_content(
-    value, path: str, policy: NetworkPolicyConfig, blocked_urls: list[str]
-):
-    if not isinstance(value, list):
-        blocked = blocked_content_path(value, path, policy, blocked_urls)
-        return ("", [blocked]) if blocked else (value, [])
-
-    mediated = []
-    capabilities = []
-    for index, block in enumerate(value):
-        item_path = f"{path}[{index}]"
-        if isinstance(block, dict) and block.get("type") in _CONTENT_WRAPPERS:
-            if blocked := blocked_content_path(
-                {**block, "content": []}, item_path, policy, blocked_urls
-            ):
-                capabilities.append(blocked)
-                continue
-            content, removed = mediate_content(
-                block.get("content"), f"{item_path}.content", policy, blocked_urls
-            )
-            if removed:
-                block["content"] = content or ""
-                capabilities.extend(removed)
-        elif blocked := blocked_content_path(block, item_path, policy, blocked_urls):
-            capabilities.append(blocked)
-            continue
-        mediated.append(block)
-    return mediated, capabilities
+            if source_kind == "file":
+                return (
+                    f"{source_path}.file_id"
+                    if source.get("file_id")
+                    else f"{source_path}.type"
+                )
+            if source_kind not in ("base64", "text", "url"):
+                return f"{source_path}.type"
+
+        if kind in (
+            "container_upload",
+            "code_execution_output",
+            "bash_code_execution_output",
+        ) and value.get("file_id"):
+            return f"{path}.file_id"
+        if kind in self.wrappers:
+            return self.blocked(value.get("content"), f"{path}.content")
+        return None if kind in _SAFE_CONTENT_TYPES else f"{path}.type"
 
 
 def content_to_wire(content) -> str | list[dict]:
@@ -464,80 +441,29 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
         self, body: RawRequest, policy: NetworkPolicyConfig
     ) -> tuple[RawRequest, list[str]]:
         mediated = body
-        capabilities: list[str] = []
-        blocked_urls: list[str] = []
+        request_filter = AnthropicRequestFilter(policy)
+        capabilities = request_filter.capabilities
 
         for key in ("container", "mcp_servers"):
             if mediated.pop(key, None):
                 capabilities.append(key)
 
-        system, removed = mediate_content(
-            mediated.get("system"), "system", policy, blocked_urls
-        )
-        capabilities.extend(removed)
-        if removed:
-            if system:
-                mediated["system"] = system
-            else:
-                mediated.pop("system")
+        if (
+            request_filter.content(mediated, "system", "system")
+            and not mediated["system"]
+        ):
+            mediated.pop("system")
 
         for message_index, message in enumerate(mediated.get("messages") or []):
             if not isinstance(message, dict):
                 continue
-            content, removed = mediate_content(
-                message.get("content"),
+            request_filter.content(
+                message,
+                "content",
                 f"messages[{message_index}].content",
-                policy,
-                blocked_urls,
             )
-            capabilities.extend(removed)
-            if removed:
-                message["content"] = content or ""
-
-        raw_tools = mediated.get("tools")
-        tool_items = raw_tools if isinstance(raw_tools, list) else []
-        if raw_tools is not None and not isinstance(raw_tools, list):
-            capabilities.append("tools")
-        tools = []
-        for index, tool in enumerate(tool_items):
-            kind = tool.get("type") if isinstance(tool, dict) else None
-            if (
-                isinstance(tool, dict)
-                and isinstance(kind, str)
-                and _WEB_TOOL_TYPE(kind)
-            ):
-                callers = tool.get("allowed_callers")
-                filter_key = "blocked_domains" if policy.block else "allowed_domains"
-                other_key = "allowed_domains" if policy.block else "blocked_domains"
-                domains = (
-                    provider_domains(policy, tool.get(filter_key))
-                    # Anthropic does not support combining allow and block filters.
-                    if tool.get(other_key) in (None, [])
-                    and (
-                        callers is None
-                        or isinstance(callers, list)
-                        and "direct" in callers
-                    )
-                    else []
-                )
-                if domains:
-                    web_tool = {
-                        **tool,
-                        filter_key: domains,
-                        "allowed_callers": ["direct"],
-                    }
-                    web_tool.pop(other_key, None)
-                    tools.append(web_tool)
-                    continue
-                capabilities.append(f"tools[{index}].type")
-                continue
-            if isinstance(tool, dict) and (
-                (isinstance(kind, str) and _CLIENT_TOOL_TYPE(kind))
-                or (kind in (None, "custom") and "input_schema" in tool)
-            ):
-                tools.append(tool)
-                continue
-            capabilities.append(f"tools[{index}].type")
+
+        tools = request_filter.tools(mediated.get("tools"))
         if "tools" in mediated:
             mediated["tools"] = tools
 
@@ -562,7 +488,8 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
 
         if capabilities:
             append_user_notice(
-                mediated.setdefault("messages", []), blocked_urls=blocked_urls
+                mediated.setdefault("messages", []),
+                blocked_urls=request_filter.blocked_urls,
             )
         return mediated, capabilities
 

{verifiers-0.3.2.dev126 → verifiers-0.3.2.dev127}/verifiers/v1/dialects/base.py

@@ -15,13 +15,17 @@ import json
 import logging
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Mapping
+from glob import has_magic
 from typing import Any, ClassVar, Generic, TypeVar
-from urllib.parse import urlsplit
 
 from pydantic import AnyHttpUrl, BaseModel, ValidationError
 from pydantic_core import from_json
 
-from verifiers.v1.configs.runtime import NetworkPolicyConfig
+from verifiers.v1.configs.runtime import (
+    NetworkPolicyConfig,
+    intersect_network_hosts,
+    parse_network_rule,
+)
 from verifiers.v1.types import Request, Response, Sampling, SamplingConfig
 
 RespT = TypeVar("RespT", bound=BaseModel)
@@ -36,21 +40,90 @@ CAPABILITY_NOTICE = (
 )
 
 
-def blocked_url(
-    value: str, policy: NetworkPolicyConfig, blocked_urls: list[str] | None = None
-) -> bool:
-    """Whether a provider-resolved resource is neither inline nor policy-permitted."""
-    if value.lower().startswith("data:"):
-        return False
-    try:
-        url = AnyHttpUrl(value)
-    except ValidationError:
-        return True
-    host = url.host.lower().rstrip(".").strip("[]")
-    blocked = not policy.permits(url.scheme, host, url.port)
-    if blocked and blocked_urls is not None:
-        blocked_urls.append(value)
-    return blocked
+class RequestFilter:
+    """One request's omissions and blocked URLs; subclasses define native wire rules."""
+
+    wrappers: tuple[str, ...] = ()
+
+    def __init__(self, policy: NetworkPolicyConfig):
+        self.policy = policy
+        self.blocked_urls: list[str] = []
+        self.capabilities: list[str] = []
+
+    def blocked_url(self, value: object) -> bool:
+        if not isinstance(value, str):
+            return True
+        if value.lower().startswith("data:"):
+            return False
+        try:
+            url = AnyHttpUrl(value)
+        except ValidationError:
+            return True
+        blocked = not self.policy.permits(url.scheme, url.host.strip("[]"), url.port)
+        if blocked:
+            self.blocked_urls.append(value)
+        return blocked
+
+    def blocked(self, value, path: str) -> str | None:
+        """Find the first forbidden part, treating nested content as one unit."""
+        if isinstance(value, list):
+            for index, item in enumerate(value):
+                if blocked := self.blocked(item, f"{path}[{index}]"):
+                    return blocked
+            return None
+        if not isinstance(value, dict):
+            return None
+        caller = value.get("caller")
+        if caller is not None and not (
+            isinstance(caller, dict) and caller.get("type") == "direct"
+        ):
+            return f"{path}.caller.type"
+        return self.blocked_part(value, path)
+
+    def blocked_part(self, value: dict, path: str) -> str | None:
+        raise NotImplementedError
+
+    def mediate(self, value, path: str):
+        """Remove forbidden parts while retaining supported wrapper blocks."""
+        if not isinstance(value, list):
+            if blocked := self.blocked(value, path):
+                self.capabilities.append(blocked)
+                return ""
+            return value
+        mediated = []
+        for index, block in enumerate(value):
+            item_path = f"{path}[{index}]"
+            wrapper = isinstance(block, dict) and block.get("type") in self.wrappers
+            scan = {**block, "content": []} if wrapper else block
+            if blocked := self.blocked(scan, item_path):
+                self.capabilities.append(blocked)
+                continue
+            if wrapper:
+                self.content(block, "content", f"{item_path}.content")
+            mediated.append(block)
+        return mediated
+
+    def content(self, parent: dict, key: str, path: str) -> bool:
+        """Rewrite a content field only when filtering removes something."""
+        before = len(self.capabilities)
+        content = self.mediate(parent.get(key), path)
+        changed = len(self.capabilities) != before
+        if changed:
+            parent[key] = content or ""
+        return changed
+
+    def tools(self, value, path: str = "tools") -> list[dict]:
+        if value is not None and not isinstance(value, list):
+            self.capabilities.append(path)
+            return []
+        tools = []
+        for index, tool in enumerate(value or []):
+            if (filtered := self.tool(tool, f"{path}[{index}]")) is not None:
+                tools.append(filtered)
+        return tools
+
+    def tool(self, value, path: str) -> dict | None:
+        raise NotImplementedError
 
 
 def provider_domains(
@@ -62,75 +135,54 @@ def provider_domains(
     Empty results mean the policy cannot be represented; never send an empty filter.
     """
     rules = policy.block or policy.allow
-    if not rules or "*" in rules:
+    if requested is not None and not isinstance(requested, list):
         return []
-    domains = []
-    hosts = []
-    for rule in rules:
-        try:
-            url = urlsplit(rule if "://" in rule else f"//{rule}")
-            port = url.port
-        except ValueError:
-            return []
-        host = (url.hostname or "").lower().rstrip(".")
-        domain = host.removeprefix("*.")
-        if (
-            url.scheme
-            or port is not None
-            or not domain
-            or any(char in domain for char in "*?[]")
-            or not domain.isascii()
-        ):
-            return []
-        hosts.append(host)
-        domains.append(domain)
+    hosts, requested_domains = [], []
+    for entries, output, is_filter in (
+        (rules, hosts, False),
+        (requested or [], requested_domains, True),
+    ):
+        for rule in entries:
+            if not isinstance(rule, str):
+                return []
+            try:
+                url, host, port = parse_network_rule(rule)
+            except ValueError:
+                return []
+            if is_filter and (
+                url.username is not None or url.path or url.query or url.fragment
+            ):
+                return []
+            domain = host if is_filter else host.removeprefix("*.")
+            if (
+                url.scheme
+                or port is not None
+                or not domain
+                or has_magic(domain)
+                or not domain.isascii()
+            ):
+                return []
+            output.append(host)
     for host in hosts:
         if not host.startswith("*.") and not (
             policy.block
             and any(
                 wildcard.startswith("*.")
-                and (host == wildcard[2:] or host.endswith(wildcard[1:]))
+                and intersect_network_hosts(wildcard, host) == host
                 for wildcard in hosts
             )
         ):
             return []
-    domains = list(dict.fromkeys(domains))
+    domains = list(dict.fromkeys(host.removeprefix("*.") for host in hosts))
     if requested is None:
         return domains
-    if not isinstance(requested, list):
-        return []
-    requested_domains = []
-    for domain in requested:
-        if not isinstance(domain, str):
-            return []
-        try:
-            url = urlsplit(domain if "://" in domain else f"//{domain}")
-            port = url.port
-        except ValueError:
-            return []
-        host = (url.hostname or "").lower().rstrip(".")
-        if (
-            url.scheme
-            or port is not None
-            or url.username is not None
-            or url.path
-            or url.query
-            or url.fragment
-            or not host
-            or any(char in host for char in "*?[]")
-            or not host.isascii()
-        ):
-            return []
-        requested_domains.append(host)
     if policy.block:
         return list(dict.fromkeys([*domains, *requested_domains]))
     intersection = []
     for allowed in domains:
         for requested_domain in requested_domains:
-            if allowed == requested_domain or allowed.endswith(f".{requested_domain}"):
-                intersection.append(allowed)
-            elif requested_domain.endswith(f".{allowed}"):
-                intersection.append(requested_domain)
+            if host := intersect_network_hosts(f"*.{allowed}", f"*.{requested_domain}"):
+                intersection.append(host.removeprefix("*."))
     return list(dict.fromkeys(intersection))