verifiers 0.3.1.dev24__tar.gz → 0.3.1.dev31__tar.gz

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev24
+Version: 0.3.1.dev31
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/pyproject.toml

@@ -76,10 +76,23 @@ dev = [
 # from this repo. The built-in tasksets/harnesses ship inside verifiers (verifiers/v1/...).
 examples = [
     "compact",
-    "gsm8k", "glossary", "deepwiki", "wiki-search", "code-golf",
-    "proposer-solver", "reverse-text", "color-codeword",
-    "wordle", "openenv-wordle", "alphabet-sort", "scratchpad",
+    "gsm8k",
+    "glossary",
+    "deepwiki",
+    "wiki-search",
+    "code-golf",
+    "proposer-solver",
+    "reverse-text",
+    "color-codeword",
+    "wordle",
+    "openenv-wordle",
+    "alphabet-sort",
+    "scratchpad",
     "kuhn-poker",
+    "interception",
+    "grayscale-interception",
+    "bash-interception",
+    "web-search-interception",
 ]
 
 [project.optional-dependencies]
@@ -164,6 +177,10 @@ color-codeword = { path = "environments/color_codeword", editable = true }
 alphabet-sort = { path = "environments/alphabet_sort", editable = true }
 scratchpad = { path = "environments/scratchpad", editable = true }
 kuhn-poker = { path = "environments/kuhn_poker", editable = true }
+interception = { path = "environments/interception", editable = true }
+grayscale-interception = { path = "environments/grayscale_interception", editable = true }
+bash-interception = { path = "environments/bash_interception", editable = true }
+web-search-interception = { path = "environments/web_search_interception", editable = true }
 
 [tool.uv.exclude-newer-package]
 # Bounded cutoffs for the explicitly requested tool upgrades.

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/tests/v1/test_scoring.py

@@ -5,6 +5,9 @@ from verifiers.v1.graph import MessageNode
 from verifiers.v1.types import AssistantMessage, UserMessage
 
 PLUGGED_FNS_PY = """
+from verifiers.v1 import Trace
+
+
 async def reply_length(trace) -> float:
     return float(len(trace.last_reply or ""))
 
@@ -17,7 +20,7 @@ async def marker(trace) -> float:
     return 0.125
 
 
-async def two_turns(trace) -> bool:
+async def two_turns(trace: Trace) -> bool:
     return trace.num_turns >= 2
 """
 

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/verifiers/v1/__init__.py

@@ -80,7 +80,9 @@ from verifiers.v1.trace import (
     Branch,
     Error,
     EvalRunInfo,
+    InterceptRecord,
     ModelCall,
+    PolicyEvent,
     Reward,
     RunInfo,
     TimeSpan,
@@ -102,6 +104,7 @@ from verifiers.v1.types import (
     Message,
     MessageContent,
     Messages,
+    Request,
     Response,
     Sampling,
     SamplingConfig,
@@ -120,7 +123,13 @@ from verifiers.v1.utils.artifacts import (
     collect,
     restore,
 )
-from verifiers.v1.utils.decorators import metric, reward, stop, tool
+from verifiers.v1.utils.decorators import (
+    intercept,
+    metric,
+    reward,
+    stop,
+    tool,
+)
 from verifiers.v1.utils.git import (
     PATCH_CAP_BYTES as PATCH_CAP_BYTES,
 )
@@ -177,6 +186,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Message",
     "MessageContent",
     "Messages",
+    "Request",
     "Response",
     "Sampling",
     "SamplingConfig",
@@ -204,6 +214,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "RunInfo",
     "EvalRunInfo",
     "ModelCall",
+    "PolicyEvent",
     "TrainRunInfo",
     "VersionInfo",
     "State",
@@ -218,10 +229,13 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "AgentSpan",
     "Error",
     # decorators
+    "intercept",
     "stop",
     "tool",
     "metric",
     "reward",
+    # interception
+    "InterceptRecord",
     # errors
     "RolloutError",
     "EnvError",

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/verifiers/v1/agent.py

@@ -19,13 +19,13 @@ from verifiers.v1.clients import (
     ModelContext,
 )
 from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
+from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects import parse_message
 from verifiers.v1.harness import Harness
 from verifiers.v1.interception import Interception, InterceptionServer
 from verifiers.v1.mcp import SharedToolServer
 from verifiers.v1.rollout import Rollout, RolloutTimeouts
 from verifiers.v1.runtimes import (
-    NetworkPolicyConfig,
     Runtime,
     RuntimeConfig,
     SubprocessConfig,
@@ -465,15 +465,11 @@ class Agent:
         interaction = Interaction(run, gate=self._gate)
         async with self._gate or nullcontext():
             opened = await run.open()
-            if not opened:
+            if not opened and (failure := run.failure) is not None:
                 trace = await run.close()
                 if trace.agent.runtime is not None:
                     trace.agent.runtime.borrowed = runtime is not None
-        if not opened:
-            failure = run.failure
-            if failure is None:  # `open()` returning False always captures one.
-                raise RuntimeError("rollout setup failed without a captured error")
-            raise failure
+                raise failure
         try:
             yield interaction
         except Exception as e:

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/verifiers/v1/clients/client.py

@@ -33,14 +33,14 @@ class Client(ABC):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
+        sampling: SamplingConfig,
         session_id: str | None = None,
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> Response:
-        """Run one completion -> a vf `Response`. The eval client forwards the native JSON
-        and parses a copy via `dialect`; the train client renders `body` to token ids.
+        """Run one completion -> a vf `Response`. `body` is the final effective native
+        request after overrides and policy mediation: the eval client forwards it unchanged,
+        while the train client renders it to token ids using the resolved `sampling` config.
         `session_id` is the rollout's trace id (sent as `SESSION_ID_HEADER`); `turn` is the
         graph-resolved prompt prefix, used by train clients for renderer bridging."""
 
@@ -48,14 +48,11 @@ class Client(ABC):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
         session_id: str | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> RelayReply:
-        """Stream a (possibly SSE) response back, relaying the provider's bytes — the proxy's
-        path for a streaming request. Only the relay (eval) client supports it; the renderer
-        generates and cannot stream."""
+        """Stream a response for the final effective `body`, relaying the provider's bytes.
+        Only the relay (eval) client supports it; the renderer generates and cannot stream."""
         raise NotImplementedError(f"{type(self).__name__} does not support streaming")
 
     async def relay_aux(

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/verifiers/v1/clients/eval.py

@@ -70,15 +70,14 @@ class EvalClient(Client):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
+        sampling: SamplingConfig,
         session_id: str | None = None,
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> Response:
         resp = await self._request(
             join_url(self.base_url, dialect.upstream_path),
-            dialect.apply_overrides(body, model, sampling_args),
+            body,
             self._headers(dialect, headers, session_id),
         )
         # A corrupted response (e.g. an HTML error page or a truncated body on a
@@ -164,8 +163,6 @@ class EvalClient(Client):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
         session_id: str | None = None,
         headers: Mapping[str, str] | None = None,
     ) -> RelayReply:
@@ -173,7 +170,7 @@ class EvalClient(Client):
         # between them. Error responses are mapped before any event is handed back.
         resp = await self._request(
             join_url(self.base_url, dialect.upstream_path),
-            dialect.apply_overrides(body, model, sampling_args),
+            body,
             self._headers(dialect, headers, session_id),
             stream=True,
         )

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/verifiers/v1/clients/train.py

@@ -319,8 +319,7 @@ class TrainClient(Client):
         self,
         dialect: Dialect,
         body: dict,
-        model: str,
-        sampling_args: SamplingConfig,
+        sampling: SamplingConfig,
         session_id: str | None = None,
         turn: PendingTurn | None = None,
         headers: Mapping[str, str] | None = None,
@@ -343,7 +342,9 @@ class TrainClient(Client):
             prompt = turn.prompt
             tools = parse_tools(body.get("tools"))
         else:
-            prompt, tools = dialect.parse_request(body)
+            request = dialect.parse_request(body)
+            prompt = request.messages
+            tools = request.tools
         from renderers.client import generate
 
         wire_tools = [tool_to_wire(t) for t in tools] if tools else None
@@ -353,7 +354,8 @@ class TrainClient(Client):
         prompt_ids: list[int] | None = None
         multi_modal_data = None
         prompt_attribution: RenderedTokens | None = None
-        raw_sampling = sampling_args.model_dump(exclude_none=True)
+        model = body["model"]
+        raw_sampling = sampling.model_dump(exclude_none=True)
         sampling_params: dict[str, Any] = dict(
             raw_sampling.pop("extra_body", None) or {}
         )

verifiers-0.3.1.dev31/verifiers/v1/configs/runtime.py

@@ -0,0 +1,66 @@
+"""Shared configuration for execution-time network policy."""
+
+from fnmatch import fnmatchcase
+from typing import Self
+from urllib.parse import urlsplit
+
+from pydantic import Field, model_validator
+from pydantic_config import BaseConfig
+
+
+def network_rule_matches(rule: str, scheme: str, host: str, port: int) -> bool:
+    """Match a network-policy host pattern or URL origin. Paths are ignored."""
+    value = rule.lower().rstrip("/")
+    try:
+        parsed = urlsplit(value if "://" in value else f"//{value}")
+        rule_port = parsed.port
+    except ValueError:
+        return False
+    pattern = (parsed.hostname or "").rstrip(".")
+    if not pattern or (parsed.scheme and parsed.scheme != scheme):
+        return False
+    if parsed.scheme and rule_port is None:
+        rule_port = 443 if parsed.scheme == "https" else 80
+    if rule_port is not None and rule_port != port:
+        return False
+    host = host.lower().rstrip(".")
+    return fnmatchcase(host, pattern) or (
+        pattern.startswith("*.") and host == pattern[2:]
+    )
+
+
+class NetworkPolicyConfig(BaseConfig):
+    """Shared execution-time policy surface for runtimes that support it."""
+
+    allow: list[str] = Field(default_factory=lambda: ["*"])
+    """Destinations allowed during execution; `*` is unrestricted and `[]` is
+    framework-only."""
+    block: list[str] = Field(default_factory=list)
+    """Destinations denied during execution; any `*` makes the policy framework-only."""
+
+    @model_validator(mode="after")
+    def validate_network_policy(self) -> Self:
+        if not self.allow or "*" in self.block:
+            # Empty allowlists and wildcard blocks both mean framework-only access.
+            self.allow = []
+            self.block = ["*"]
+        elif self.allow != ["*"] and self.block:
+            raise ValueError(
+                "non-empty concrete allow and block egress lists are mutually exclusive"
+            )
+        return self
+
+    @property
+    def network_restricted(self) -> bool:
+        return "*" not in self.allow or bool(self.block)
+
+    def with_task_network_policy(self, allow: list[str], block: list[str]) -> Self:
+        values = self.model_dump()
+        if "*" in allow:
+            allow = self.allow
+        elif "*" not in self.allow:
+            # Rules are opaque: guessing overlap between different globs could widen the policy.
+            runtime_allow = set(self.allow)
+            allow = [rule for rule in allow if rule in runtime_allow]
+        block = list(dict.fromkeys([*block, *self.block]))
+        return type(self).model_validate({**values, "allow": allow, "block": block})

{verifiers-0.3.1.dev24 → verifiers-0.3.1.dev31}/verifiers/v1/configs/task.py

@@ -40,9 +40,9 @@ class TaskConfig(BaseConfig):
     """Judge plugins run after task rewards, set through `--env.taskset.task.judges`."""
 
     stops: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
-    """Stop conditions `(trace) -> bool` plugged by name, merged with the task's
-    `@vf.stop` methods (a plugged function replaces a decorated one with the same
-    name)."""
+    """Typed `Request`, `Response`, or `Trace` stop predicates plugged by name and
+    merged with the task's `@vf.stop` methods (a plugged function replaces a
+    decorated one with the same name)."""
     metrics: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
     """Metrics `(task, trace, runtime) -> float` plugged by name, merged with the
     task's `@vf.metric` methods."""