verifiers 0.3.1.dev42__tar.gz → 0.3.1.dev52__tar.gz

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.1.dev42
+Version: 0.3.1.dev52
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -34,7 +34,7 @@ Requires-Dist: msgpack>=1.1.2
 Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai-agents>=0.8.2
 Requires-Dist: openai>=2.9.0
-Requires-Dist: prime-pydantic-config[toml]>=0.4.2
+Requires-Dist: prime-pydantic-config[toml]>=0.4.3
 Requires-Dist: prime-sandboxes>=0.2.37
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/pyproject.toml

@@ -47,7 +47,7 @@ dependencies = [
     "setproctitle>=1.3.0",
     "httpx>=0.27.0",
     "aiohttp>=3.9.0",
-    "prime-pydantic-config[toml]>=0.4.2",
+    "prime-pydantic-config[toml]>=0.4.3",
     "uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/tests/v1/conftest.py

@@ -154,6 +154,8 @@ def _eval_config(
         runtime_cfg = dict(runtime)
         _configure_prime_runtimes(runtime_cfg)
         env_cfg.setdefault("agent", {})["runtime"] = runtime_cfg
+    retries = {"max_retries": 2, "include": ["ProviderError", "HarnessError"]}
+    env_cfg.setdefault("retries", retries)
     # Per-run caps live on the seats: resolve the env's declared roles and cap
     # each one (a test's own seat dict wins over the shared defaults).
     config_cls = vf.env_config_type(taskset, env_cfg.get("id", ""))
@@ -167,11 +169,8 @@ def _eval_config(
         seat_cfg.setdefault("max_turns", max_turns)
         seat_cfg.setdefault("max_output_tokens", max_tokens)
         seat_cfg.setdefault("timeout", {"rollout": rollout_timeout, "scoring": 60})
-        # Flake resilience: retries are per-agent now (flat RetryConfig).
-        seat_cfg.setdefault(
-            "retries",
-            {"max_retries": 2, "include": ["ProviderError", "HarnessError"]},
-        )
+        # Agent runs retry locally; interactions retry with their whole episode.
+        seat_cfg.setdefault("retries", retries)
     return EvalConfig(
         env={
             "taskset": taskset_cfg,

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/tests/v1/test_judges.py

@@ -184,7 +184,7 @@ def test_judges_reject_shared_reward_keys():
 
 
 def test_reward_name_fallback():
-    # name > id's package name > snake-cased class name (code-level judge with neither).
+    # name > id > snake-cased class name (code-level judge with neither).
     class MyQualityJudge(vf.Judge[float]):
         prompt = "{x}"
 
@@ -192,7 +192,7 @@ def test_reward_name_fallback():
         vf.ReferenceJudge().reward_name == "reference"
     )  # class-name fallback (no id set)
     assert (
-        vf.ReferenceJudge(vf.ReferenceJudgeConfig(id="org/my-judge@1.0.0")).reward_name
+        vf.ReferenceJudge(vf.ReferenceJudgeConfig(id="my-judge")).reward_name
         == "my-judge"
     )
     assert vf.ReferenceJudge(vf.ReferenceJudgeConfig(name="gold")).reward_name == "gold"

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/agent.py

@@ -548,6 +548,8 @@ class Agent:
             else self.runtime_config
         )
         async with provision_runtime(config) as runtime:
+            # Keep sandbox startup task-neutral: this box may later host another task.
+            runtime.env = dict(task.runtime_env()) if task is not None else {}
             yield runtime
 
 

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/cli/dashboard/eval.py

@@ -24,7 +24,6 @@ from verifiers.v1.utils.format import (
     format_override,
     format_time,
 )
-from verifiers.v1.utils.install import env_name
 from verifiers.v1.utils.interrupt import cleaning_up
 
 if TYPE_CHECKING:
@@ -240,7 +239,7 @@ def Overview(config: EvalConfig) -> Table:
     taskset = config.env.taskset
     env_label = taskset.name if taskset.id else "no taskset"
     if config.env.id:
-        env_label = f"{env_name(config.env.id)}+{env_label}"
+        env_label = f"{config.env.id}+{env_label}"
     # One seat story when every seat resolves the same way (the common case); one
     # row per seat when they diverge (a judge on its own harness/runtime).
     runtimes = {role: getattr(config.env, role).runtime.type for role in seats}

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/cli/debug.py

@@ -224,6 +224,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         else task.data.timeout.setup
     )
     try:
+        runtime.env = dict(task.runtime_env())
         trace.timing.boot.start = time.time()
         await runtime.start()
         now = time.time()

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/cli/validate.py

@@ -235,6 +235,7 @@ async def _run_gold(task: Task, config: ValidateConfig) -> ResultRow:
     )
     valid, exc = False, None
     try:
+        runtime.env = dict(task.runtime_env())
         trace = Trace(
             task=TraceTask(
                 type=type(task).__name__, data=task.data, key=task.key, hash=task.hash
@@ -278,6 +279,7 @@ async def _run_setup(task: Task, config: ValidateConfig) -> ResultRow:
     )
     valid, exc = False, None
     try:
+        runtime.env = dict(task.runtime_env())
         trace = Trace(
             task=TraceTask(
                 type=type(task).__name__, data=task.data, key=task.key, hash=task.hash

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/configs/cli/eval.py

@@ -12,7 +12,6 @@ from verifiers.v1.configs.env import EnvConfig
 from verifiers.v1.configs.serve import ServeConfig
 from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
 from verifiers.v1.types import SamplingConfig
-from verifiers.v1.utils.install import env_name
 
 
 def default_run_name(env: EnvConfig, model: str) -> str:
@@ -24,7 +23,7 @@ def default_run_name(env: EnvConfig, model: str) -> str:
     if taskset.id and env.id:
         # Same compounding as `EnvConfig.env_id`: a `best-of-n+gsm8k` run must
         # not share a name with a plain `gsm8k` one.
-        name = f"{env_name(env.id)}+{name}"
+        name = f"{env.id}+{name}"
     # Every seat's resolved harness, distinct, in role order.
     harness = "+".join(dict.fromkeys(h.name for h in env.agent_harnesses().values()))
     slug = (

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/configs/harness.py

@@ -9,12 +9,11 @@ from pydantic import ConfigDict, Field, FiniteFloat
 from pydantic_config import BaseConfig
 
 from verifiers.v1.types import ID
-from verifiers.v1.utils.install import env_name
 
 
 class HarnessConfig(BaseConfig):
     id: ID = "bash"
-    """Local package or Hub `org/name[@version]`, set through the seat's
+    """Installed harness package, set through the seat's
     `--env.<role>.harness.id` (`--env.agent.harness.id` on the single-agent env)."""
     env: dict[str, str] = Field(default_factory=dict)
     """Extra program variables; harness-owned variables take precedence."""
@@ -30,7 +29,7 @@ class HarnessConfig(BaseConfig):
 
     @property
     def name(self) -> str:
-        return env_name(self.id)
+        return self.id
 
     @property
     def resolved_env(self) -> dict[str, str]:

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/configs/judge.py

@@ -9,7 +9,6 @@ from pydantic import BaseModel, FiniteFloat, SerializeAsAny
 
 from verifiers.v1.clients import BaseClientConfig
 from verifiers.v1.types import ID, SamplingConfig
-from verifiers.v1.utils.install import env_name
 
 
 class JudgeConfig(BaseClientConfig):
@@ -29,7 +28,7 @@ Judges = list[SerializeAsAny[JudgeConfig]]
 
 
 def judge_key(config: JudgeConfig) -> str:
-    return config.name or env_name(config.id)
+    return config.name or config.id
 
 
 def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
@@ -41,7 +40,7 @@ def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
         if not raw.get("id"):
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, a local package, or a hub `org/name` package)"
+                "`rubric`, or an installed package)"
             )
         resolved.append(judge_config_type(raw["id"]).model_validate(raw))
     return resolved
@@ -52,7 +51,7 @@ def check_judges(entries: Sequence[JudgeConfig]) -> None:
         if not entry.id:
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, a local package, or a hub `org/name` package)"
+                "`rubric`, or an installed package)"
             )
     keys = [judge_key(entry) for entry in entries]
     if duplicates := {key for key in keys if keys.count(key) > 1}:

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/configs/taskset.py

@@ -7,12 +7,11 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.types import ID
-from verifiers.v1.utils.install import env_name
 
 
 class TasksetConfig(BaseConfig):
     id: ID = ""
-    """Local package or Hub `org/name[@version]`, set with `--env.taskset.id` (or the
+    """Installed taskset package, set with `--env.taskset.id` (or the
     positional `eval <taskset-id>`)."""
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
@@ -22,4 +21,4 @@ class TasksetConfig(BaseConfig):
 
     @property
     def name(self) -> str:
-        return env_name(self.id)
+        return self.id

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/dialects/anthropic.py

@@ -16,12 +16,14 @@ from anthropic.types import Message as AnthropicMessage
 from anthropic.types import MessageCreateParams
 from anthropic.types import Usage as AnthropicUsage
 
+from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects.base import (
     Dialect,
     StreamParser,
     append_user_notice,
     blocked_url,
     parse_sse_event,
+    provider_allowed_domains,
 )
 from verifiers.v1.types import (
     AssistantMessage,
@@ -56,6 +58,7 @@ THINKING = ("redacted_thinking", "thinking")
 # at the provider. Anchoring the pattern keeps new versions client-side without treating an
 # arbitrary dated provider tool as safe.
 _CLIENT_TOOL_TYPE = re.compile(r"(?:bash|text_editor|computer|memory)_\d{8}").fullmatch
+_WEB_TOOL_TYPE = re.compile(r"web_(?:search|fetch)_\d{8}").fullmatch
 _CONTENT_WRAPPERS = (
     "tool_result",
     "code_execution_tool_result",
@@ -126,10 +129,10 @@ def parse_content(content) -> str | list[ContentPart]:
     return parts
 
 
-def blocked_content_path(value, path: str) -> str | None:
+def blocked_content_path(value, path: str, policy: NetworkPolicyConfig) -> str | None:
     if isinstance(value, list):
         for index, item in enumerate(value):
-            if blocked := blocked_content_path(item, f"{path}[{index}]"):
+            if blocked := blocked_content_path(item, f"{path}[{index}]", policy):
                 return blocked
         return None
     if not isinstance(value, dict):
@@ -148,10 +151,12 @@ def blocked_content_path(value, path: str) -> str | None:
             return source_path
         source_kind = source.get("type")
         if source_kind == "content":
-            return blocked_content_path(source.get("content"), f"{source_path}.content")
+            return blocked_content_path(
+                source.get("content"), f"{source_path}.content", policy
+            )
         if source_kind == "url":
             url = source.get("url")
-            if not isinstance(url, str) or blocked_url(url):
+            if not isinstance(url, str) or blocked_url(url, policy):
                 return f"{source_path}.url"
         if source_kind == "file":
             return (
@@ -169,13 +174,13 @@ def blocked_content_path(value, path: str) -> str | None:
     ) and value.get("file_id"):
         return f"{path}.file_id"
     if kind in _CONTENT_WRAPPERS:
-        return blocked_content_path(value.get("content"), f"{path}.content")
+        return blocked_content_path(value.get("content"), f"{path}.content", policy)
     return None if kind in _SAFE_CONTENT_TYPES else f"{path}.type"
 
 
-def mediate_content(value, path: str):
+def mediate_content(value, path: str, policy: NetworkPolicyConfig):
     if not isinstance(value, list):
-        blocked = blocked_content_path(value, path)
+        blocked = blocked_content_path(value, path, policy)
         return ("", [blocked]) if blocked else (value, [])
 
     mediated = []
@@ -183,16 +188,18 @@ def mediate_content(value, path: str):
     for index, block in enumerate(value):
         item_path = f"{path}[{index}]"
         if isinstance(block, dict) and block.get("type") in _CONTENT_WRAPPERS:
-            if blocked := blocked_content_path({**block, "content": []}, item_path):
+            if blocked := blocked_content_path(
+                {**block, "content": []}, item_path, policy
+            ):
                 capabilities.append(blocked)
                 continue
             content, removed = mediate_content(
-                block.get("content"), f"{item_path}.content"
+                block.get("content"), f"{item_path}.content", policy
             )
             if removed:
                 block["content"] = content or ""
                 capabilities.extend(removed)
-        elif blocked := blocked_content_path(block, item_path):
+        elif blocked := blocked_content_path(block, item_path, policy):
             capabilities.append(blocked)
             continue
         mediated.append(block)
@@ -448,7 +455,7 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
     response_type = ModdedAnthropicMessage
 
     def mediate_external_capabilities(
-        self, body: MessageCreateParams
+        self, body: MessageCreateParams, policy: NetworkPolicyConfig
     ) -> tuple[MessageCreateParams, list[str]]:
         mediated = body
         capabilities: list[str] = []
@@ -457,7 +464,7 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
             if mediated.pop(key, None):
                 capabilities.append(key)
 
-        system, removed = mediate_content(mediated.get("system"), "system")
+        system, removed = mediate_content(mediated.get("system"), "system", policy)
         capabilities.extend(removed)
         if removed:
             if system:
@@ -469,7 +476,9 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
             if not isinstance(message, dict):
                 continue
             content, removed = mediate_content(
-                message.get("content"), f"messages[{message_index}].content"
+                message.get("content"),
+                f"messages[{message_index}].content",
+                policy,
             )
             capabilities.extend(removed)
             if removed:
@@ -482,6 +491,33 @@ class AnthropicDialect(Dialect[MessageCreateParams, AnthropicMessage]):
         tools = []
         for index, tool in enumerate(tool_items):
             kind = tool.get("type") if isinstance(tool, dict) else None
+            if (
+                isinstance(tool, dict)
+                and isinstance(kind, str)
+                and _WEB_TOOL_TYPE(kind)
+            ):
+                callers = tool.get("allowed_callers")
+                domains = (
+                    provider_allowed_domains(policy, tool.get("allowed_domains"))
+                    if tool.get("blocked_domains") in (None, [])
+                    and (
+                        callers is None
+                        or isinstance(callers, list)
+                        and "direct" in callers
+                    )
+                    else []
+                )
+                if domains:
+                    web_tool = {
+                        **tool,
+                        "allowed_domains": domains,
+                        "allowed_callers": ["direct"],
+                    }
+                    web_tool.pop("blocked_domains", None)
+                    tools.append(web_tool)
+                    continue
+                capabilities.append(f"tools[{index}].type")
+                continue
             if isinstance(tool, dict) and (
                 (isinstance(kind, str) and _CLIENT_TOOL_TYPE(kind))
                 or (kind in (None, "custom") and "input_schema" in tool)

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/dialects/base.py

@@ -16,16 +16,19 @@ import logging
 from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterator, Mapping
 from typing import ClassVar, Generic, TypeVar
+from urllib.parse import urlsplit
 
-from pydantic import BaseModel
+from pydantic import AnyHttpUrl, BaseModel, TypeAdapter, ValidationError
 from pydantic_core import from_json
 
+from verifiers.v1.configs.runtime import NetworkPolicyConfig, network_rule_matches
 from verifiers.v1.types import Request, Response, Sampling, SamplingConfig
 
 ReqT = TypeVar("ReqT")
 RespT = TypeVar("RespT", bound=BaseModel)
 
 logger = logging.getLogger(__name__)
+HTTP_URL_ADAPTER = TypeAdapter(AnyHttpUrl)
 
 PROVIDER_CAPABILITY_POLICY_CODE = "provider_capability_unavailable"
 CAPABILITY_NOTICE = (
@@ -35,9 +38,83 @@ CAPABILITY_NOTICE = (
 )
 
 
-def blocked_url(value: str) -> bool:
-    """Whether a provider-resolved resource is not inline data."""
-    return not value.lower().startswith("data:")
+def blocked_url(value: str, policy: NetworkPolicyConfig) -> bool:
+    """Whether a provider-resolved resource is neither inline nor policy-permitted."""
+    if value.lower().startswith("data:"):
+        return False
+    try:
+        url = HTTP_URL_ADAPTER.validate_python(value)
+    except ValidationError:
+        return True
+    host = url.host.lower().rstrip(".").strip("[]")
+    return any(
+        network_rule_matches(rule, url.scheme, host, url.port) for rule in policy.block
+    ) or not any(
+        network_rule_matches(rule, url.scheme, host, url.port) for rule in policy.allow
+    )
+
+
+def provider_allowed_domains(
+    policy: NetworkPolicyConfig, requested: object = None
+) -> list[str]:
+    """Translate a network policy to provider domain-filter semantics without widening it."""
+    if policy.block or not policy.allow or "*" in policy.allow:
+        return []
+    domains = []
+    for rule in policy.allow:
+        try:
+            url = urlsplit(rule if "://" in rule else f"//{rule}")
+            port = url.port
+        except ValueError:
+            return []
+        host = (url.hostname or "").lower().rstrip(".")
+        domain = host.removeprefix("*.")
+        if (
+            url.scheme
+            or port is not None
+            or not host.startswith("*.")
+            or not domain
+            or "*" in domain
+            or not domain.isascii()
+        ):
+            return []
+        domains.append(domain)
+    domains = list(dict.fromkeys(domains))
+    if requested is None:
+        return domains
+    if not isinstance(requested, list):
+        return []
+    requested_domains = []
+    for domain in requested:
+        if not isinstance(domain, str):
+            return []
+        try:
+            url = urlsplit(domain if "://" in domain else f"//{domain}")
+            port = url.port
+        except ValueError:
+            return []
+        host = (url.hostname or "").lower().rstrip(".")
+        if (
+            url.scheme
+            or port is not None
+            or url.username is not None
+            or url.path
+            or url.query
+            or url.fragment
+            or not host
+            or "*" in host
+            or not host.isascii()
+        ):
+            return []
+        requested_domains.append(host)
+    intersection = []
+    for allowed in domains:
+        for requested_domain in requested_domains:
+            if allowed == requested_domain or allowed.endswith(f".{requested_domain}"):
+                intersection.append(allowed)
+            elif requested_domain.endswith(f".{allowed}"):
+                intersection.append(requested_domain)
+    return list(dict.fromkeys(intersection))
 
 
 def append_user_notice(
@@ -187,7 +264,9 @@ class Dialect(ABC, Generic[ReqT, RespT]):
         return {"error": {"message": message, "type": "invalid_request_error"}}
 
     @abstractmethod
-    def mediate_external_capabilities(self, body: ReqT) -> tuple[ReqT, list[str]]:
+    def mediate_external_capabilities(
+        self, body: ReqT, policy: NetworkPolicyConfig
+    ) -> tuple[ReqT, list[str]]:
         """Remove provider-side capabilities during restricted execution. Implementations add
         the same policy context on every call because the agent does not retain injected request
         content. Returned paths never contain request values."""

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev52}/verifiers/v1/dialects/chat.py

@@ -17,6 +17,7 @@ from urllib.parse import urlsplit
 from openai.types.chat import ChatCompletion
 from openai.types.chat.completion_create_params import CompletionCreateParams
 
+from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects.base import (
     Dialect,
     StreamParser,
@@ -356,7 +357,7 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
     response_type = ModdedChatCompletion
 
     def mediate_external_capabilities(
-        self, body: CompletionCreateParams
+        self, body: CompletionCreateParams, policy: NetworkPolicyConfig
     ) -> tuple[CompletionCreateParams, list[str]]:
         mediated = body
         capabilities: list[str] = []
@@ -446,7 +447,7 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
                 elif kind == "image_url":
                     image = part.get("image_url") or {}
                     url = image.get("url") if isinstance(image, dict) else image
-                    if not isinstance(url, str) or blocked_url(url):
+                    if not isinstance(url, str) or blocked_url(url, policy):
                         capability = f"{path}.image_url.url"
                 elif kind == "file":
                     file = part.get("file")
@@ -465,7 +466,7 @@ class ChatDialect(Dialect[CompletionCreateParams, ChatCompletion]):
                                 capability = f"{path}.file.file_data"
                             else:
                                 if (parsed.scheme or parsed.netloc) and blocked_url(
-                                    file_data
+                                    file_data, policy
                                 ):
                                     capability = f"{path}.file.file_data"
                 if capability is None: