verifiers 0.4.1.dev48__tar.gz → 0.4.1.dev50__tar.gz

{verifiers-0.4.1.dev48 → verifiers-0.4.1.dev50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev48
+Version: 0.4.1.dev50
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.4.1.dev50/tests/unit/test_harbor_grading_network.py

@@ -0,0 +1,49 @@
+"""Which network a parsed Harbor task's shared-box verifier grades under."""
+
+import pytest
+
+pytest.importorskip("harbor")
+
+import verifiers.v1 as vf
+from verifiers.v1.tasksets.harbor.taskset import HarborConfig, parse_task
+
+
+def task(tmp_path, *, environment=None, agent=None, verifier=None):
+    def section(name, mode):
+        return f"[{name}]\nnetwork_mode = {mode!r}\n" if mode else f"[{name}]\n"
+
+    (tmp_path / "instruction.md").write_text("Do nothing.")
+    (tmp_path / "tests").mkdir(exist_ok=True)
+    (tmp_path / "tests" / "test.sh").write_text("echo 1 > /logs/verifier/reward.txt\n")
+    (tmp_path / "task.toml").write_text(
+        'version = "1.0"\n'
+        + section("environment", environment).replace(
+            "[environment]\n", '[environment]\ndocker_image = "python:3.11-slim"\n'
+        )
+        + section("agent", agent)
+        + section("verifier", verifier)
+    )
+    return parse_task(tmp_path, 0, HarborConfig())
+
+
+def test_an_offline_task_grades_offline(tmp_path):
+    data = task(
+        tmp_path, environment="no-network", agent="no-network", verifier="no-network"
+    )
+    assert data.scoring.mode == "shared"
+    assert data.scoring.network == vf.NetworkPolicyConfig(allow=[])
+
+
+def test_the_environment_baseline_applies_without_a_verifier_policy(tmp_path):
+    data = task(tmp_path, environment="no-network")
+    assert data.scoring.network == vf.NetworkPolicyConfig(allow=[])
+
+
+def test_a_public_verifier_opens_the_box(tmp_path):
+    assert task(tmp_path, agent="no-network").scoring.network is None
+    assert task(tmp_path, agent="no-network", verifier="public").scoring.network is None
+
+
+def test_a_restricted_verifier_beside_a_public_agent_grades_open(tmp_path, caplog):
+    assert task(tmp_path, verifier="no-network").scoring.network is None
+    assert "cannot restrict a box that started open" in caplog.text

{verifiers-0.4.1.dev48 → verifiers-0.4.1.dev50}/verifiers/v1/cli/replay.py

@@ -1,7 +1,8 @@
 """Offline scoring for saved traces.
 
-Replay clears each trace's scores and recomputes everything computable from the saved
-transcript — trace-only handlers plus the layered config's judges. Runtime-requiring
+Replay re-scores each trace with `Task.rescore`: its scores cleared, everything
+computable from the saved transcript recomputed — trace-only handlers plus the layered
+config's judges. Runtime-requiring
 signals don't run offline, so a replay carries offline scores only; the source run keeps
 the runtime-recorded values. Its saved config is the base for replay-specific overrides.
 Multi-agent runs don't support replay: re-scoring is per trace, and the env's
@@ -152,16 +153,13 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
                 st.state, st.detail, st.end = "skipped", "rollout errored", time.time()
             else:
                 st.state = "running"
-                # Clear the recorded scores; only offline-recomputable ones come back.
-                trace.info.pop("judge_calls", None)
-                trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
                 # The declared Task for a rebuilt row, the base Task for the
                 # WireTaskData fallback.
                 task = (task_cls if isinstance(row, data_cls) else Task)(
                     row, config=config.taskset.task
                 )
                 try:
-                    await task.score(trace)
+                    await task.rescore(trace)
                     st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
                 except Exception as exc:
                     st.state, st.detail = "error", type(exc).__name__

{verifiers-0.4.1.dev48 → verifiers-0.4.1.dev50}/verifiers/v1/task.py

@@ -224,6 +224,15 @@ class Task(Generic[DataT, StateT, ConfigT]):
         as a rollout would."""
         return None
 
+    async def rescore(self, trace: Trace) -> None:
+        """Score a finished trace again, offline: clear its rewards, metrics and judge
+        calls, then run this task's signals that need only the trace."""
+        if trace.has_error:
+            raise ValueError("the rollout errored: its transcript is incomplete")
+        trace.info.pop("judge_calls", None)
+        trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
+        await self.score(trace)
+
     def defer_scoring(self) -> Self:
         """An independent copy whose task signals are deferred.
 

{verifiers-0.4.1.dev48 → verifiers-0.4.1.dev50}/verifiers/v1/tasksets/harbor/taskset.py

@@ -550,6 +550,29 @@ def harbor_network(network) -> NetworkPolicyConfig:
     return NetworkPolicyConfig(allow=list(network.allowed_hosts))
 
 
+def shared_grading_network(parsed) -> NetworkPolicyConfig | None:
+    """The network a shared-box verifier grades under: Harbor's verifier phase policy,
+    or the `[environment]` baseline when the phase declares none. None opens the box for
+    grading, as for a public verifier. A restricted verifier beside a public agent phase
+    also grades open: its box started without a policy, so it cannot take one."""
+    from harbor.models.task.config import NetworkMode
+
+    baseline = parsed.environment.resolve_baseline()
+    grading = parsed.verifier.explicit_phase_policy() or baseline
+    if grading.network_mode == NetworkMode.PUBLIC:
+        return None
+    agent = parsed.agent.explicit_phase_policy() or baseline
+    if agent.network_mode == NetworkMode.PUBLIC:
+        logger.warning(
+            "Harbor task declares a %s verifier beside a public agent phase; shared "
+            "grading cannot restrict a box that started open, so it grades with open "
+            "network",
+            grading.network_mode.value,
+        )
+        return None
+    return harbor_network(grading)
+
+
 def task_resources(environment, multiplier: float) -> TaskResources:
     """Harbor environment resource requests, scaled, as `TaskResources`.
 
@@ -781,7 +804,10 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
         )
     return HarborData(
         idx=idx,
-        scoring=ScoringConfig(mode="isolated" if verifier is not None else "shared"),
+        scoring=ScoringConfig(
+            mode="isolated" if verifier is not None else "shared",
+            network=None if verifier is not None else shared_grading_network(parsed),
+        ),
         artifact_max_bytes=harbor_config.artifact_max_bytes,
         name=harbor_task.name,
         description=task.description if task else None,