verifiers 0.4.1.dev46__tar.gz → 0.4.1.dev48__tar.gz

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev46
+Version: 0.4.1.dev48
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/tests/integration/fixtures/network_policy.py

@@ -63,9 +63,9 @@ class NetworkPolicyTaskset(vf.Taskset[NetworkPolicyTask, vf.TasksetConfig]):
     def load(self) -> list[NetworkPolicyTask]:
         policies = {
             "open": {},
-            "off": {"network_allow": []},
-            "block": {"network_block": [BLOCKED]},
-            "allow": {"network_allow": [ALLOWED]},
+            "off": {"network": vf.NetworkPolicyConfig(allow=[])},
+            "block": {"network": vf.NetworkPolicyConfig(block=[BLOCKED])},
+            "allow": {"network": vf.NetworkPolicyConfig(allow=[ALLOWED])},
         }
         prompt = (
             f"Your only task: run the command `python3 {PROBE_PATH}` once in your "

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/tests/integration/grid.py

@@ -192,31 +192,32 @@ def grid(
     return cells
 
 
-def transports(harness: str) -> tuple[str, ...]:
-    """The wire formats the harness config can select (`transport`) besides its
-    default. Empty when the program speaks one format or picks its own."""
-    field = harness_config_type(harness).model_fields.get("transport")
-    if field is None:
-        return ()
-    return tuple(t for t in get_args(field.annotation) if t != field.default)
+def dialects(harness: str) -> tuple[str, ...]:
+    """The dialects the harness config can select besides its default. Empty when the
+    program speaks one dialect or picks its own."""
+    field = harness_config_type(harness).model_fields["dialect"]
+    return tuple(
+        value
+        for arg in get_args(field.annotation)
+        for value in get_args(arg) or (arg,)
+        if value != field.default
+    )
 
 
-def transport_grid(
+def dialect_grid(
     runtimes: Sequence[str] = ("prime",), *, needs: tuple[str, ...] = ()
 ) -> list:
-    """`grid` cells for every non-default `transport` each harness config can select."""
+    """`grid` cells for every non-default `dialect` each harness config can select."""
     cells = []
     for harness in HARNESSES:
         name = harness_id(harness)
-        for transport in transports(name):
-            for cell in grid(
-                [{"id": name, "transport": transport}], runtimes, needs=needs
-            ):
+        for dialect in dialects(name):
+            for cell in grid([{"id": name, "dialect": dialect}], runtimes, needs=needs):
                 cells.append(
                     pytest.param(
                         *cell.values,
                         marks=cell.marks,
-                        id=cell.id.replace(f"{name}-in-", f"{name}-{transport}-in-"),
+                        id=cell.id.replace(f"{name}-in-", f"{name}-{dialect}-in-"),
                     )
                 )
     return cells

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/tests/integration/test_multi_turn.py

@@ -1,14 +1,14 @@
 """One task, many turns: the model completes a shell task (echo-agentic: write a phrase
 to a file, checked in the runtime) across the harness x runtime grid, and on prime over
-every other `transport` a harness config can select. A turn the harness re-sends
+every other `dialect` a harness config can select. A turn the harness re-sends
 differently, beyond what its `replay_view` declares, forks the trace."""
 
 import pytest
-from grid import grid, mark, transport_grid
+from grid import dialect_grid, grid, mark
 
 MULTI_TURN = [
     *grid(needs=("shell",)),
-    *transport_grid(needs=("shell",)),
+    *dialect_grid(needs=("shell",)),
     pytest.param(
         "bash",
         {"type": "modal", "allow": []},

verifiers-0.4.1.dev48/tests/integration/test_scoring_network.py

@@ -0,0 +1,175 @@
+"""Scoring runs under the network its policy declares: a task with framework-only egress
+probes two public hosts from its scoring hook, in the workspace (shared) and in a fresh
+verifier (isolated), in rollouts and in vf-validate, and a Harbor task's trusted grading
+lifts the policy unless scoring declares its network."""
+
+import json
+import shutil
+
+import pytest
+
+import verifiers.v1 as vf
+from verifiers.v1.cli.validate import _run_check
+from verifiers.v1.configs.agent import TimeoutConfig
+from verifiers.v1.configs.cli.validate import ValidateConfig
+from verifiers.v1.tasksets.harbor.taskset import HarborData, HarborTask
+
+pytestmark = [
+    pytest.mark.docker,
+    pytest.mark.skipif(not shutil.which("docker"), reason="needs Docker"),
+]
+
+IMAGE = "python:3.11-slim"
+HOSTS = ["example.com", "pypi.org"]
+# Prints which hosts answer 200. Requests go through the runtime's egress proxy, which
+# refuses a host the policy denies.
+PROBE = f"""
+import json, urllib.request
+reachable = []
+for host in {HOSTS!r}:
+    try:
+        with urllib.request.urlopen("https://" + host, timeout=15) as response:
+            if response.status == 200:
+                reachable.append(host)
+    except Exception:
+        pass
+print(json.dumps(reachable))
+"""
+
+
+class IdleHarness(vf.Harness):
+    async def launch(self, ctx, trace, runtime, endpoint, secret, mcp_urls, data):
+        return await runtime.run_program(["true"], {})
+
+
+class ProbeTask(vf.Task):
+    @vf.reward(weight=1.0)
+    async def probe(self, trace, runtime) -> float:
+        result = await runtime.run(["python", "-c", PROBE], {})
+        trace.info["reachable"] = json.loads(result.stdout)
+        return 1.0
+
+
+async def run(task) -> vf.Trace:
+    config = vf.AgentConfig(
+        model="unused",
+        runtime=vf.DockerConfig(),
+        timeout=TimeoutConfig(setup=180, finalize=30, scoring=120),
+    )
+    async with vf.Agent(config) as agent:
+        agent.harness = IdleHarness(vf.HarnessConfig())
+        return await agent.run(task)
+
+
+@pytest.mark.parametrize("mode", ["shared", "isolated"])
+@pytest.mark.parametrize(
+    ("network", "reachable"),
+    [
+        (None, []),
+        (vf.NetworkPolicyConfig(allow=["example.com"]), ["example.com"]),
+        (vf.NetworkPolicyConfig(), HOSTS),
+    ],
+    ids=["undeclared", "one-host", "open"],
+)
+async def test_scoring_network(mode, network, reachable):
+    task = ProbeTask(
+        vf.TaskData(
+            prompt="Do nothing",
+            image=IMAGE,
+            network=vf.NetworkPolicyConfig(allow=[]),
+            scoring=vf.ScoringConfig(
+                mode=mode, network=network, retries=vf.RetryConfig(max_retries=0)
+            ),
+        )
+    )
+    trace = await run(task)
+    assert trace.ok, trace.errors
+    assert trace.info["reachable"] == reachable
+
+
+class ReachTask(vf.Task):
+    @vf.reward(weight=1.0)
+    async def reach(self, trace, runtime) -> float:
+        result = await runtime.run(["python", "-c", PROBE], {})
+        return float(json.loads(result.stdout) == ["example.com"])
+
+
+@pytest.mark.parametrize("mode", ["shared", "isolated"])
+async def test_validate_scores_under_the_declared_network(mode):
+    task = ReachTask(
+        vf.TaskData(
+            prompt="Do nothing",
+            image=IMAGE,
+            network=vf.NetworkPolicyConfig(allow=[]),
+            scoring=vf.ScoringConfig(
+                mode=mode,
+                network=vf.NetworkPolicyConfig(allow=["example.com"]),
+                retries=vf.RetryConfig(max_retries=0),
+            ),
+        )
+    )
+    row = await _run_check(task, ValidateConfig(runtime=vf.DockerConfig()), "noop")
+    # vf-validate never applies the agent phase's policy, so the shared workspace is open
+    # until scoring declares a network. Scoring that reaches only example.com passes the
+    # untouched task, which the noop check reports as invalid.
+    assert row["error"] is None
+    assert row["valid"] is False
+
+
+class GoldReachTask(ReachTask):
+    async def validate(self, trace, runtime) -> bool:
+        await vf.grade_submission(self, trace, runtime)
+        return trace.reward == 1.0
+
+
+async def test_validate_grades_a_submission_under_the_declared_network():
+    task = GoldReachTask(
+        vf.TaskData(
+            prompt="Do nothing",
+            image=IMAGE,
+            network=vf.NetworkPolicyConfig(allow=[]),
+            scoring=vf.ScoringConfig(
+                mode="isolated",
+                network=vf.NetworkPolicyConfig(allow=["example.com"]),
+                retries=vf.RetryConfig(max_retries=0),
+            ),
+        )
+    )
+    # vf-validate's trace records its own runtime config, so the verifier can reach
+    # example.com although the task's policy is framework-only.
+    row = await _run_check(task, ValidateConfig(runtime=vf.DockerConfig()), "gold")
+    assert row["error"] is None
+    assert row["valid"] is True
+
+
+@pytest.mark.parametrize(
+    ("network", "reward"),
+    [(None, 1.0), (vf.NetworkPolicyConfig(allow=["pypi.org"]), 0.0)],
+    ids=["trusted-default", "declared"],
+)
+async def test_harbor_grading_network(tmp_path, network, reward):
+    tests = tmp_path / "tests"
+    tests.mkdir()
+    (tests / "test.sh").write_text(
+        "python - <<'EOF'\n"
+        "import urllib.request\n"
+        "try:\n"
+        "    urllib.request.urlopen('https://example.com', timeout=15)\n"
+        "    reward = 1\n"
+        "except Exception:\n"
+        "    reward = 0\n"
+        "open('/logs/verifier/reward.txt', 'w').write(str(reward))\n"
+        "EOF\n"
+    )
+    task = HarborTask(
+        HarborData(
+            prompt="Do nothing",
+            task_dir=str(tmp_path),
+            image=IMAGE,
+            network=vf.NetworkPolicyConfig(allow=[]),
+            scoring=vf.ScoringConfig(network=network),
+        )
+    )
+    trace = await run(task)
+    assert trace.ok, trace.errors
+    assert trace.reward == reward

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/tests/integration/test_tool_vms.py

@@ -69,7 +69,9 @@ async def test_a_tool_vm_keeps_its_own_egress_beside_a_restricted_workspace(
         runtime=vf.PrimeConfig(labels=["vf-integration"], allow=allow)
     )
     task = ProbeTask(
-        vf.TaskData(prompt="Probe the network", network_allow=[]),
+        vf.TaskData(
+            prompt="Probe the network", network=vf.NetworkPolicyConfig(allow=[])
+        ),
         FetchToolTaskConfig(tools=tools),
     )
     config = vf.AgentConfig(

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/tests/unit/test_graph.py

@@ -305,6 +305,14 @@ def _replayed_trace(replay_view, sampled, replayed, dialect=None):
         ),
         (strip_text, {"content": "Calling it.\n\n"}, {"content": "Calling it."}),
         (strip_text, {"content": None}, {"reasoning_content": " "}),
+        (
+            strip_text,
+            {"content": "\n\n"},
+            {
+                "reasoning_content": " ",
+                "provider_state": [{"type": "thinking", "thinking": " "}],
+            },
+        ),
     ],
 )
 def test_replay_view_matches_cosmetically_rewritten_turns(view, sampled, replayed):

verifiers-0.4.1.dev48/tests/unit/test_scoring_network.py

@@ -0,0 +1,113 @@
+"""Scoring's declared network: how agent and task policies combine, how it combines with
+the runtime's own rules (in `grade_submission`, the agent's), and what the runtime asks
+its provider for."""
+
+from typing import cast
+
+import pytest
+
+import verifiers.v1 as vf
+from verifiers.v1 import grading
+from verifiers.v1.configs.scoring import resolve_scoring
+from verifiers.v1.errors import SandboxError, TaskError
+from verifiers.v1.grading import grade_submission, scoring_network
+from verifiers.v1.runtimes import deployment
+from verifiers.v1.runtimes.base import RuntimeBackend
+from verifiers.v1.runtimes.runtime import Runtime
+from verifiers.v1.utils.compile import resolve_runtime_config
+
+PYPI = vf.NetworkPolicyConfig(allow=["pypi.org"])
+
+
+def test_agent_network_replaces_the_tasks():
+    task = vf.ScoringConfig(network=PYPI)
+    assert resolve_scoring(task, vf.ScoringConfig()).network == PYPI
+    agent = vf.ScoringConfig(network=vf.NetworkPolicyConfig())
+    assert resolve_scoring(task, agent).network == vf.NetworkPolicyConfig()
+
+
+def test_scoring_network_combines_with_the_runtimes_rules():
+    assert scoring_network(vf.DockerConfig(), PYPI).allow == ["pypi.org"]
+    narrowed = vf.DockerConfig(allow=["pypi.org", "example.com"])
+    opened = scoring_network(narrowed, vf.NetworkPolicyConfig())
+    assert opened.allow == ["pypi.org", "example.com"]
+    # As for `TaskData.network`, a concrete allowlist can't sit beside a blocklist.
+    with pytest.raises(ValueError):
+        scoring_network(vf.DockerConfig(block=["example.com"]), PYPI)
+
+
+def test_scoring_network_needs_a_runtime_with_policies():
+    with pytest.raises(TaskError):
+        scoring_network(vf.SubprocessConfig(), PYPI)
+    open_network = vf.NetworkPolicyConfig()
+    assert scoring_network(vf.SubprocessConfig(), open_network) is open_network
+
+
+class Backend:
+    supports_user = False
+    name = "fake"
+
+    def __init__(self, config, restricted=True):
+        self.config = config
+        self.network_restricted = restricted
+        self.calls = []
+
+    async def prepare_execution(self, routes, network=None):
+        self.calls.append((routes, None if network is None else network.allow))
+
+
+async def test_set_network_enforces_a_policy_or_lifts_it():
+    backend = Backend(vf.DockerConfig(allow=[]))
+    runtime = Runtime(cast(RuntimeBackend, backend))
+    await runtime.set_network(PYPI)
+    await runtime.set_network(vf.NetworkPolicyConfig())
+    assert backend.calls == [([], ["pypi.org"]), (None, None)]
+
+
+async def test_set_network_cannot_restrict_a_runtime_that_started_open():
+    backend = Backend(vf.DockerConfig(), restricted=False)
+    runtime = Runtime(cast(RuntimeBackend, backend))
+    with pytest.raises(SandboxError):
+        await runtime.set_network(PYPI)
+    await runtime.set_network(vf.NetworkPolicyConfig())
+    assert backend.calls == [(None, None)]
+
+
+async def test_grade_submission_widens_from_the_agents_runtime(monkeypatch):
+    graded = []
+
+    async def grade_task(task, trace, config, *args, **kwargs):
+        graded.append(config)
+
+    class Deployment:
+        def __init__(self, *args, **kwargs):
+            pass
+
+        async def start(self):
+            pass
+
+        async def close(self):
+            pass
+
+    monkeypatch.setattr(grading, "grade_task", grade_task)
+    monkeypatch.setattr(deployment, "Deployment", Deployment)
+    task = vf.Task(
+        vf.TaskData(
+            prompt="",
+            image="python:3.11-slim",
+            network=vf.NetworkPolicyConfig(allow=[]),
+            scoring=vf.ScoringConfig(mode="isolated", network=PYPI),
+        )
+    )
+    agent_runtime = vf.DockerConfig()
+    # The solver's resolved config carries the task's framework-only policy.
+    solver = Runtime(
+        cast(RuntimeBackend, Backend(resolve_runtime_config(agent_runtime, task)))
+    )
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig(runtime=agent_runtime)),
+        task=vf.TraceTask(type="Task", data=task.data),
+    )
+    await grade_submission(task, trace, solver)
+    [config] = graded
+    assert config.allow == ["pypi.org"]

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/tests/unit/utils/test_loaders.py

@@ -29,10 +29,10 @@ def test_every_builtin_harness_loads() -> None:
 
 def test_harness_config_narrows_to_its_own_type() -> None:
     config = vf.AgentConfig.model_validate(
-        {"harness": {"id": "kimi-code", "transport": "responses"}}
+        {"harness": {"id": "kimi-code", "dialect": "responses"}}
     )
     assert type(config.harness) is harness_config_type("kimi-code")
-    assert config.harness.transport == "responses"
+    assert config.harness.dialect == "responses"
 
 
 def test_unknown_harness_names_the_builtins() -> None:

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/__init__.py

@@ -82,6 +82,7 @@ from verifiers.v1.mcp import (
 from verifiers.v1.runtimes import (
     ApptainerConfig,
     DockerConfig,
+    NetworkPolicyConfig,
     PodmanConfig,
     PrimeConfig,
     ProgramResult,
@@ -315,6 +316,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "ProgramResult",
     "SubprocessConfig",
     "DockerConfig",
+    "NetworkPolicyConfig",
     "PodmanConfig",
     "ApptainerConfig",
     "PrimeConfig",

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/agent.py

@@ -94,7 +94,7 @@ def _check_borrowed_placement(
     """A borrowed box is never re-provisioned, so a task's placement fields can't
     be honored. Reject requirements that cannot be applied to the running box; an
     image mismatch on a container only warns, since sharing its world is the point."""
-    task_policy = "*" not in task.data.network_allow or bool(task.data.network_block)
+    task_policy = task.data.network.network_restricted
     base_policy = base_config if isinstance(base_config, NetworkPolicyConfig) else None
     if task_policy or (base_policy is not None and base_policy.network_restricted):
         config = runtime.config

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/cli/validate.py

@@ -31,7 +31,12 @@ from verifiers.v1.cli.resolve import (
 from verifiers.v1.cli.resume import distribute
 from verifiers.v1.configs.cli.validate import ValidateConfig
 from verifiers.v1.errors import TaskError, boundary
-from verifiers.v1.grading import bind_scoring_env, grade_task, verifier_runtime
+from verifiers.v1.grading import (
+    bind_scoring_env,
+    grade_task,
+    scoring_network,
+    verifier_runtime,
+)
 from verifiers.v1.runtimes import make_backend
 from verifiers.v1.runtimes.deployment import Deployment
 from verifiers.v1.state import state_cls
@@ -244,7 +249,7 @@ async def _setup_only(task: Task, trace: Trace, runtime) -> bool:
 
 async def _gold(task: Task, trace: Trace, runtime) -> bool | None:
     """Apply and check the reference answer."""
-    return await task.validate(runtime)
+    return await invoke(task.validate, {"trace": trace, "runtime": runtime})
 
 
 async def _score_untouched(
@@ -288,6 +293,10 @@ async def _score_untouched(
         async with asyncio.timeout(task.data.timeout.scoring):
             runtime = bind_scoring_env(runtime, policy)
             await invoke(grader.stage_verifier, {"trace": trace, "runtime": runtime})
+            if policy.network is not None:
+                await runtime.set_network(
+                    scoring_network(config.runtime, policy.network)
+                )
             await grader.score(trace, runtime, deployment)
     if not trace.rewards:
         return None

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/configs/harness.py

@@ -9,7 +9,7 @@ from typing import Annotated, Literal
 from pydantic import ConfigDict, Field, FiniteFloat
 from pydantic_config import BaseConfig
 
-from verifiers.v1.types import ID
+from verifiers.v1.types import ID, DialectName
 
 PinnedVersion = Annotated[str, Field(pattern=r"^[A-Za-z0-9._+-]+$")]
 """A release/tag a harness pins its program install to."""
@@ -52,6 +52,9 @@ class HarnessConfig(BaseConfig):
     tool_timeout: FiniteFloat = Field(600.0, gt=0)
     """Seconds a single MCP tool call may take; raise it for tools that boot a VM."""
     disabled_tools: list[str] | None = None
+    dialect: DialectName | Literal["auto"] = "auto"
+    """Model API wire format the program speaks. `auto` lets the program pick, e.g. from
+    the model. Each harness narrows this to the formats it can select."""
     skills: list[SkillSource] = Field(default_factory=list)
     """Host skill folders or `{runtime: path}` roots of skills inside the runtime.
     Sources are installed in order; later files override matching earlier files.

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/configs/runtime.py

@@ -55,7 +55,9 @@ def intersect_network_hosts(left: str, right: str) -> str:
 
 
 class NetworkPolicyConfig(BaseConfig):
-    """Shared execution-time policy surface for runtimes that support it."""
+    """What a box may reach. A runtime's own rules, a task's (`TaskData.network`) and a
+    task's scoring (`ScoringConfig.network`) all use it; a task's policy combines with its
+    runtime's rules: allowlists intersect and blocklists merge."""
 
     allow: list[str] = Field(default_factory=lambda: ["*"])
     """Destinations allowed during execution; `*` is unrestricted and `[]` is
@@ -87,10 +89,12 @@ class NetworkPolicyConfig(BaseConfig):
             for rule in self.block
         ) and any(network_rule_matches(rule, scheme, host, port) for rule in self.allow)
 
-    def with_task_network_policy(self, allow: list[str], block: list[str]) -> Self:
+    def with_task_network_policy(self, network: "NetworkPolicyConfig") -> Self:
         values = self.model_dump()
         # Intersection must not erase syntax the runtime cannot enforce.
-        task = type(self).model_validate({**values, "allow": allow, "block": block})
+        task = type(self).model_validate(
+            {**values, "allow": network.allow, "block": network.block}
+        )
         allow, block = task.allow, task.block
         if "*" in allow:
             allow = self.allow

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/configs/scoring.py

@@ -6,6 +6,7 @@ from pydantic import SerializationInfo, field_serializer
 from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.retries import RetryConfig
+from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.runtimes import RuntimeConfig
 from verifiers.v1.utils.generic import merge_defaults
 
@@ -19,6 +20,11 @@ class ScoringConfig(BaseConfig):
     """Scoring-only variables, merged over task defaults and resolved at scoring time."""
     retries: RetryConfig = RetryConfig(max_retries=2)
     """Retry rules for fresh isolated scoring attempts; shared scoring is not retried."""
+    network: NetworkPolicyConfig | None = None
+    """The task's network policy while scoring runs, in place of `TaskData.network` and
+    combined with the runtime's own rules the same way. None keeps the default: isolated
+    scoring runs under `TaskData.network`, shared scoring keeps the agent phase's network,
+    and Harbor's shared grading lifts the policy."""
 
     @field_serializer("runtime")
     def serialize_runtime(

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/envs/agentic_judge/env.py

@@ -165,8 +165,7 @@ class JudgeTask(vf.Task):
                 image=solved.image,
                 workdir=solved.workdir,
                 resources=solved.resources,
-                network_allow=solved.network_allow,
-                network_block=solved.network_block,
+                network=solved.network,
             ),
             files=files,
         )
@@ -334,9 +333,8 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
                 "workdir": runtime.config.workdir,
             }
             if isinstance(runtime.config, NetworkPolicyConfig):
-                updates.update(
-                    network_allow=runtime.config.allow,
-                    network_block=runtime.config.block,
+                updates["network"] = NetworkPolicyConfig(
+                    allow=runtime.config.allow, block=runtime.config.block
                 )
             judge_task = judge_task.with_data(**updates)
             # A per-invocation view retains episode accounting and concurrency while

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/grading.py

@@ -9,6 +9,7 @@ from pathlib import PurePosixPath
 from verifiers.v1.configs.scoring import ScoringConfig
 from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.runtimes import (
+    NetworkPolicyConfig,
     PrimeConfig,
     Runtime,
     RuntimeConfig,
@@ -56,6 +57,8 @@ def verifier_runtime(
         image=task.data.image or getattr(solver_runtime.config, "image", None),
         workdir=task.data.workdir or getattr(solver_runtime.config, "workdir", None),
     )
+    if policy.network is not None:
+        task = task.with_data(network=policy.network)
     config = resolve_runtime_config(source, task)
     if policy.runtime is not None and "image" in policy.runtime.model_fields_set:
         config = config.model_copy(update={"image": policy.runtime.image})
@@ -75,6 +78,21 @@ def verifier_runtime(
     return config
 
 
+def scoring_network(
+    base: RuntimeConfig, network: NetworkPolicyConfig
+) -> NetworkPolicyConfig:
+    """`network` as the task's policy while scoring, combined with the runtime's own
+    rules the way `TaskData.network` is."""
+    if isinstance(base, NetworkPolicyConfig):
+        return base.with_task_network_policy(network)
+    if network.network_restricted:
+        raise TaskError(
+            f"scoring requires a network policy, but the {base.type} runtime does not "
+            "support one"
+        )
+    return network
+
+
 def bind_scoring_env(runtime: Runtime, policy: ScoringConfig) -> Runtime:
     """Bind scoring variables without changing the solver's runtime or backend."""
     return runtime.with_env({**runtime.env, **resolve_env(policy.env or {})})
@@ -188,6 +206,9 @@ async def grade_submission(
 
     The caller finalizes the submission and collects artifacts into the trace first.
     Uses the task's grader, scoring policy, and timeouts, including retry behavior.
+    As in a rollout, the verifier derives from the agent's runtime config on the trace
+    (`trace.agent.config.runtime`), not from the solver's resolved one, so the solver
+    task's requirements, such as its network policy, don't carry over.
     """
     from verifiers.v1.runtimes.deployment import Deployment
 
@@ -195,7 +216,7 @@ async def grade_submission(
     if policy.mode != "isolated":
         raise ValueError("grade_submission requires isolated scoring")
     grader = task.grading_task(runtime, scoring=policy)
-    config = verifier_runtime(grader, runtime, runtime.config, policy)
+    config = verifier_runtime(grader, runtime, trace.agent.config.runtime, policy)
     deployment = Deployment(
         runtime.backend, borrowed_runtime=runtime, env=dict(runtime.env)
     )

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/harnesses/bash/harness.py

@@ -1,4 +1,5 @@
 import os
+from typing import Literal
 
 from pydantic import PositiveInt
 from pydantic_config import BaseConfig
@@ -38,6 +39,7 @@ class CompactionConfig(BaseConfig):
 
 
 class BashHarnessConfig(HarnessConfig):
+    dialect: Literal["chat_completions"] = "chat_completions"
     edit: bool = True
     """Offer the local `edit` tool (single-occurrence string replacement in a file) alongside
     `bash`. On by default; set `--env.agent.harness.edit false` for a bash-only agent."""

{verifiers-0.4.1.dev46 → verifiers-0.4.1.dev48}/verifiers/v1/harnesses/browser_use/harness.py

@@ -32,6 +32,7 @@ Finding elements: prefer the accessibility tree over screenshots. cdp("Accessibi
 
 
 class BrowserUseHarnessConfig(HarnessConfig):
+    dialect: Literal["chat_completions"] = "chat_completions"
     browser: Literal["chromium", "cdp"] = "chromium"
     """`chromium` launches locally; `cdp` attaches to `cdp_url` without owning it."""