verifiers 0.4.1.dev12__tar.gz → 0.4.1.dev13__tar.gz

{verifiers-0.4.1.dev12 → verifiers-0.4.1.dev13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev12
+Version: 0.4.1.dev13
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.4.1.dev13/tests/integration/fixtures/network_policy_v1.py

@@ -0,0 +1,80 @@
+"""network-policy-v1: the agent runs an egress probe under the task's network policy.
+
+One task per policy the task data can declare: `open` (the default), `off` (an empty
+allowlist), `block` (blocks `example.com`) and `allow` (allows only `pypi.org`). Setup
+stages a probe script while egress is still open; the prompt asks the model to run it.
+The probe fetches both hosts and records which ones it reached, so the reward is 1 only
+if the policy held: each host answered exactly when the policy permits it. A harness
+without a shell writes no report and scores 0. Needs a runtime with network policy
+support; the integration test runs it on prime.
+"""
+
+import json
+
+import verifiers.v1 as vf
+from verifiers.v1.runtimes import Runtime
+
+__all__ = ["NetworkPolicyTaskset"]
+
+BLOCKED = "example.com"
+ALLOWED = "pypi.org"
+PROBE_PATH = "/tmp/network-probe.py"
+REPORT = "/tmp/network-probe.json"
+
+# What each task's policy lets through, keyed by task name.
+EXPECTED = {
+    "open": {"blocked": "ok", "allowed": "ok"},
+    "off": {"blocked": "denied", "allowed": "denied"},
+    "block": {"blocked": "denied", "allowed": "ok"},
+    "allow": {"blocked": "denied", "allowed": "ok"},
+}
+
+# Any HTTP answer, error status included, means the destination was reachable; a
+# refused connection, a proxy refusal, a DNS failure, or a timeout means it was not.
+PROBE = f"""
+import json, urllib.error, urllib.request
+report = {{}}
+for name, url in {{"blocked": "https://{BLOCKED}", "allowed": "https://{ALLOWED}/simple/"}}.items():
+    try:
+        urllib.request.urlopen(url, timeout=15)
+        report[name] = "ok"
+    except urllib.error.HTTPError:
+        report[name] = "ok"
+    except Exception:
+        report[name] = "denied"
+open({REPORT!r}, "w").write(json.dumps(report))
+print(json.dumps(report))
+"""
+
+
+class NetworkPolicyTask(vf.Task):
+    async def setup(self, trace: vf.Trace, runtime: Runtime) -> None:
+        await runtime.write(PROBE_PATH, PROBE.encode())
+
+    @vf.reward(weight=1.0)
+    async def policy_held(self, trace: vf.Trace, runtime: Runtime) -> float:
+        result = await runtime.run(["cat", REPORT], {})
+        report = json.loads(result.stdout) if result.exit_code == 0 else None
+        trace.info["network_report"] = report
+        return float(report == EXPECTED[self.data.name])
+
+
+class NetworkPolicyTaskset(vf.Taskset[NetworkPolicyTask, vf.TasksetConfig]):
+    def load(self) -> list[NetworkPolicyTask]:
+        policies = {
+            "open": {},
+            "off": {"network_allow": []},
+            "block": {"network_block": [BLOCKED]},
+            "allow": {"network_allow": [ALLOWED]},
+        }
+        prompt = (
+            f"Use your shell to run `python3 {PROBE_PATH}` once, then reply with its "
+            "output."
+        )
+        return [
+            NetworkPolicyTask(
+                vf.TaskData(idx=idx, name=name, prompt=prompt, **policy),
+                self.config.task,
+            )
+            for idx, (name, policy) in enumerate(policies.items())
+        ]

verifiers-0.4.1.dev13/tests/integration/test_network_policy.py

@@ -0,0 +1,68 @@
+"""A task's network policy holds during execution, and the model is told about it: the
+agent runs an egress probe under each policy the task data declares
+(network-policy-v1), with the agent's `network_notice` on or off, across every harness
+on prime."""
+
+import pytest
+from grid import grid
+
+from verifiers.v1.utils.loaders import harness_class
+
+NOTICES = {
+    "open": None,
+    "off": "External network access is disabled.",
+    "block": (
+        "External network access is allowed except for these blocked destinations: "
+        "example.com. Do not circumvent this selection of domains."
+    ),
+    "allow": (
+        "External network access is limited to these destinations: pypi.org. "
+        "Do not circumvent this selection of domains."
+    ),
+}
+
+
+def prompt_text(trace) -> str:
+    """Every prompt-supplied message the model saw, as one text."""
+    texts = []
+    for node in trace.nodes:
+        if node.sampled:
+            continue
+        content = node.message.content
+        if isinstance(content, str):
+            texts.append(content)
+        elif content:
+            texts.extend(getattr(part, "text", "") or "" for part in content)
+    return "\n".join(texts)
+
+
+@pytest.mark.parametrize("notice", [True, False], ids=["notice-on", "notice-off"])
+@pytest.mark.parametrize(
+    "harness,harness_runtime", grid(runtimes=["prime"]), indirect=True
+)
+async def test_network_policy(run, harness, harness_runtime, notice):
+    episodes = await run(
+        "network-policy-v1",
+        harness=harness,
+        runtime=harness_runtime,
+        num_tasks=len(NOTICES),
+        max_turns=6,
+        max_tokens=4096,
+        env={"agent": {"network_notice": notice}},
+    )
+    by_name = {episode.task.data.name: episode for episode in episodes}
+    assert set(by_name) == set(NOTICES)
+    for name, episode in by_name.items():
+        [trace] = episode.traces
+        assert trace.ok, [error.message for error in trace.errors]
+        prompt = prompt_text(trace)
+        if notice and NOTICES[name]:
+            assert NOTICES[name] in prompt, name
+        else:
+            assert "External network access" not in prompt, name
+        if harness_class(harness.id).EXECUTES_CODE:
+            assert trace.reward == 1.0, (
+                name,
+                trace.info["network_report"],
+                trace.last_reply,
+            )

verifiers-0.4.1.dev12/tests/integration/fixtures/network_policy_v1.py

@@ -1,88 +0,0 @@
-"""network-policy-v1: a scripted, model-free agent probes egress under the task's policy.
-
-Task 0 blocks `example.com`; task 1 allows only `pypi.org`. During execution the scripted
-harness fetches both hosts and the framework's own interception endpoint from inside the
-runtime and records which ones it reached. The reward is 1 only if the policy held: the
-denied host failed, the permitted host answered, and the framework route stayed reachable.
-Needs a runtime with network policy support (docker or prime).
-"""
-
-import json
-
-import verifiers.v1 as vf
-from verifiers.v1.clients import ModelContext
-from verifiers.v1.configs.harness import HarnessConfig
-from verifiers.v1.harness import Harness
-from verifiers.v1.runtimes import ProgramResult, Runtime
-
-__all__ = ["NetworkPolicyTaskset", "ScriptedNetworkProbeHarness"]
-
-BLOCKED = "example.com"
-ALLOWED = "pypi.org"
-REPORT = "/tmp/network-policy-report.json"
-
-# Any HTTP answer, error status included, means the destination was reachable; a
-# refused connection, a proxy refusal, a DNS failure, or a timeout means it was not.
-PROBE = """
-import json, sys, urllib.error, urllib.request
-report = {}
-for name, url in json.loads(sys.argv[1]).items():
-    try:
-        urllib.request.urlopen(url, timeout=15)
-        report[name] = "ok"
-    except urllib.error.HTTPError:
-        report[name] = "ok"
-    except Exception:
-        report[name] = "denied"
-open(sys.argv[2], "w").write(json.dumps(report))
-"""
-
-
-class ScriptedNetworkProbeHarness(Harness[HarnessConfig]):
-    async def launch(
-        self,
-        ctx: ModelContext,
-        trace: vf.Trace,
-        runtime: Runtime,
-        endpoint: str,
-        secret: str,
-        mcp_urls: dict[str, str],
-        data: vf.TaskData,
-        tool_interception_url: str | None = None,
-    ) -> ProgramResult:
-        urls = {
-            "blocked": f"https://{BLOCKED}",
-            "allowed": f"https://{ALLOWED}/simple/",
-            "framework": endpoint,
-        }
-        return await runtime.run_program(
-            ["python3", "-c", PROBE, json.dumps(urls), REPORT], {}
-        )
-
-
-class NetworkPolicyTask(vf.Task):
-    @vf.reward(weight=1.0)
-    async def policy_held(self, trace: vf.Trace, runtime: Runtime) -> float:
-        report = json.loads(await runtime.read(REPORT))
-        trace.info["network_report"] = report
-        return float(
-            report == {"blocked": "denied", "allowed": "ok", "framework": "ok"}
-        )
-
-
-class NetworkPolicyTaskset(vf.Taskset[NetworkPolicyTask, vf.TasksetConfig]):
-    def load(self) -> list[NetworkPolicyTask]:
-        return [
-            NetworkPolicyTask(
-                vf.TaskData(
-                    idx=0, name="block", prompt="probe", network_block=[BLOCKED]
-                ),
-                self.config.task,
-            ),
-            NetworkPolicyTask(
-                vf.TaskData(
-                    idx=1, name="allow", prompt="probe", network_allow=[ALLOWED]
-                ),
-                self.config.task,
-            ),
-        ]

verifiers-0.4.1.dev12/tests/integration/test_network_policy.py

@@ -1,28 +0,0 @@
-"""A task's network policy holds during execution: a scripted agent probes a blocked host,
-a permitted host, and the framework's own route from inside the runtime
-(network-policy-v1), across the runtimes with a network policy."""
-
-import pytest
-from grid import runtimes_grid
-
-
-@pytest.mark.parametrize(
-    "harness_runtime", runtimes_grid(needs=("network",)), indirect=True
-)
-async def test_network_policy(run, harness_runtime):
-    episodes = await run(
-        "network-policy-v1",
-        harness="network-policy-v1",
-        runtime=harness_runtime,
-        num_tasks=2,
-    )
-    traces = [trace for episode in episodes for trace in episode.traces]
-    by_name = {trace.task.data.name: trace for trace in traces}
-    assert set(by_name) == {"block", "allow"}
-    for name, trace in by_name.items():
-        assert trace.ok, [error.message for error in trace.errors]
-        assert trace.info["network_report"] == {
-            "blocked": "denied",
-            "allowed": "ok",
-            "framework": "ok",
-        }, name