verifiers 0.4.1.dev48__tar.gz → 0.4.1.dev49__tar.gz

{verifiers-0.4.1.dev48 → verifiers-0.4.1.dev49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev48
+Version: 0.4.1.dev49
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.4.1.dev49/tests/unit/test_harbor_grading_network.py

@@ -0,0 +1,49 @@
+"""Which network a parsed Harbor task's shared-box verifier grades under."""
+
+import pytest
+
+pytest.importorskip("harbor")
+
+import verifiers.v1 as vf
+from verifiers.v1.tasksets.harbor.taskset import HarborConfig, parse_task
+
+
+def task(tmp_path, *, environment=None, agent=None, verifier=None):
+    def section(name, mode):
+        return f"[{name}]\nnetwork_mode = {mode!r}\n" if mode else f"[{name}]\n"
+
+    (tmp_path / "instruction.md").write_text("Do nothing.")
+    (tmp_path / "tests").mkdir(exist_ok=True)
+    (tmp_path / "tests" / "test.sh").write_text("echo 1 > /logs/verifier/reward.txt\n")
+    (tmp_path / "task.toml").write_text(
+        'version = "1.0"\n'
+        + section("environment", environment).replace(
+            "[environment]\n", '[environment]\ndocker_image = "python:3.11-slim"\n'
+        )
+        + section("agent", agent)
+        + section("verifier", verifier)
+    )
+    return parse_task(tmp_path, 0, HarborConfig())
+
+
+def test_an_offline_task_grades_offline(tmp_path):
+    data = task(
+        tmp_path, environment="no-network", agent="no-network", verifier="no-network"
+    )
+    assert data.scoring.mode == "shared"
+    assert data.scoring.network == vf.NetworkPolicyConfig(allow=[])
+
+
+def test_the_environment_baseline_applies_without_a_verifier_policy(tmp_path):
+    data = task(tmp_path, environment="no-network")
+    assert data.scoring.network == vf.NetworkPolicyConfig(allow=[])
+
+
+def test_a_public_verifier_opens_the_box(tmp_path):
+    assert task(tmp_path, agent="no-network").scoring.network is None
+    assert task(tmp_path, agent="no-network", verifier="public").scoring.network is None
+
+
+def test_a_restricted_verifier_beside_a_public_agent_grades_open(tmp_path, caplog):
+    assert task(tmp_path, verifier="no-network").scoring.network is None
+    assert "cannot restrict a box that started open" in caplog.text

{verifiers-0.4.1.dev48 → verifiers-0.4.1.dev49}/verifiers/v1/tasksets/harbor/taskset.py

@@ -550,6 +550,29 @@ def harbor_network(network) -> NetworkPolicyConfig:
     return NetworkPolicyConfig(allow=list(network.allowed_hosts))
 
 
+def shared_grading_network(parsed) -> NetworkPolicyConfig | None:
+    """The network a shared-box verifier grades under: Harbor's verifier phase policy,
+    or the `[environment]` baseline when the phase declares none. None opens the box for
+    grading, as for a public verifier. A restricted verifier beside a public agent phase
+    also grades open: its box started without a policy, so it cannot take one."""
+    from harbor.models.task.config import NetworkMode
+
+    baseline = parsed.environment.resolve_baseline()
+    grading = parsed.verifier.explicit_phase_policy() or baseline
+    if grading.network_mode == NetworkMode.PUBLIC:
+        return None
+    agent = parsed.agent.explicit_phase_policy() or baseline
+    if agent.network_mode == NetworkMode.PUBLIC:
+        logger.warning(
+            "Harbor task declares a %s verifier beside a public agent phase; shared "
+            "grading cannot restrict a box that started open, so it grades with open "
+            "network",
+            grading.network_mode.value,
+        )
+        return None
+    return harbor_network(grading)
+
+
 def task_resources(environment, multiplier: float) -> TaskResources:
     """Harbor environment resource requests, scaled, as `TaskResources`.
 
@@ -781,7 +804,10 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
         )
     return HarborData(
         idx=idx,
-        scoring=ScoringConfig(mode="isolated" if verifier is not None else "shared"),
+        scoring=ScoringConfig(
+            mode="isolated" if verifier is not None else "shared",
+            network=None if verifier is not None else shared_grading_network(parsed),
+        ),
         artifact_max_bytes=harbor_config.artifact_max_bytes,
         name=harbor_task.name,
         description=task.description if task else None,