verifiers 0.2.2.dev84__py3-none-any.whl → 0.2.2.dev86__py3-none-any.whl

verifiers/v1/envs/agentic_judge/__init__.py

@@ -1,16 +1,16 @@
 from verifiers.v1.envs.agentic_judge.env import (
-    AgenticJudgeEnv,
     AgenticJudgeEnvConfig,
     Criterion,
+    IsolatedAgenticJudgeEnv,
     JudgeTaskConfig,
     ScoreConfig,
     TextFile,
 )
 
 __all__ = [
-    "AgenticJudgeEnv",
     "AgenticJudgeEnvConfig",
     "Criterion",
+    "IsolatedAgenticJudgeEnv",
     "JudgeTaskConfig",
     "ScoreConfig",
     "TextFile",

verifiers/v1/envs/agentic_judge/env.py

@@ -1,16 +1,17 @@
-"""agentic-judge: a solver plays the task, then a judge verifies the work.
-
-A reusable env (`--env.id agentic-judge` over any taskset). The solver plays the
-task in a container provisioned from its runtime policy; the judge then grades
-rubric criteria (`[env.task]`: policy prompt, criteria file) and writes its
-verdicts to `/tmp/verdict.json`, with the solver's full trace record uploaded at
-`/tmp/trace.json`. `finalize()` validates them strictly onto the solver's trace —
-`judge/<name>` metrics plus a weighted-mean `judge` reward, composed with the
-taskset's own rewards via `[env.score]` (judge-only by default).
-
-`--env.share-runtime` controls whether the judge uses the solver's runtime. It is
-enabled by default. When disabled, the judge gets a fresh runtime containing the
-task's collected artifacts.
+"""Agentic judging: a solver plays the task, then a judge verifies the work.
+
+Two reusable envs share the grading protocol. `--env.id agentic-judge` provisions
+a fresh box from the solver's runtime policy and restores only the task's collected
+artifacts; `--env.id shared-agentic-judge` explicitly runs the judge in the
+solver's box. The judge grades rubric criteria (`[env.task]`: policy prompt,
+criteria file) and writes its verdicts to `/tmp/verdict.json`, with the solver's
+full trace record uploaded at `/tmp/trace.json`. `finalize()` validates them
+strictly onto the solver's trace — `judge/<name>` metrics plus a weighted-mean
+`judge` reward, composed with the taskset's own rewards via `[env.score]`
+(judge-only by default).
+
+The environment id selects the runtime boundary; there is no mode boolean whose
+value can disagree with the environment's security and artifact semantics.
 """
 
 import json
@@ -165,6 +166,8 @@ class JudgeTask(vf.Task):
                 image=solved.image,
                 workdir=solved.workdir,
                 resources=solved.resources,
+                network_allow=solved.network_allow,
+                network_block=solved.network_block,
             ),
             files=files,
             artifacts={} if share_runtime else solution.state.artifacts,
@@ -258,20 +261,22 @@ class AgenticJudgeEnvConfig(vf.EnvConfig):
     """The solver agent. Its runtime must be a container:
     `--env.solver.runtime.type docker|prime`."""
     judge: vf.AgentConfig = vf.AgentConfig()
-    """The judge agent. Its runtime is ignored when `share_runtime` is enabled;
-    otherwise it must be a container."""
-    share_runtime: bool = True
-    """Whether the judge grades in the solver's runtime."""
+    """The judge agent. Its runtime setting is ignored: both judging modes use the
+    solver's resolved runtime policy, either by borrowing its box or provisioning a
+    fresh equivalent one."""
     task: JudgeTaskConfig = JudgeTaskConfig()
     score: ScoreConfig = ScoreConfig()
 
 
 class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
+    """Common agentic-judge protocol; subclasses choose the runtime boundary."""
+
     def __init__(self, config: AgenticJudgeEnvConfig) -> None:
-        if config.share_runtime:
-            config.judge = config.judge.model_copy(
-                update={"runtime": config.solver.runtime}
-            )
+        # Both modes use the solver's policy. Shared judging borrows that exact box;
+        # isolated judging resolves the mirrored JudgeTask into a fresh equivalent.
+        config.judge = config.judge.model_copy(
+            update={"runtime": config.solver.runtime}
+        )
         super().__init__(config)
         self._check_agents()
         # A missing policy file or a malformed rubric fails here, not mid-episode.
@@ -302,21 +307,6 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         # The judge grades the policy; its tokens are never training data.
         agents.judge.trainable = False
 
-    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
-        if self.config.share_runtime:
-            async with agents.solver.provision(task) as box:
-                solution = await agents.solver.run(task, runtime=box)
-                judge_task = JudgeTask.from_trace(solution, self.config.task)
-                await agents.judge.run(judge_task, runtime=box)
-            return
-
-        solution = await agents.solver.run(task)
-        if not solution.ok:
-            return
-        await agents.judge.run(
-            JudgeTask.from_trace(solution, self.config.task, share_runtime=False)
-        )
-
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
         by_agent = {t.agent.name: t for t in episode.traces}
         if "judge" not in by_agent:
@@ -334,3 +324,25 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         total = sum(criterion.weight for criterion in criteria)
         reward = sum(c.weight * scores[c.name] for c in criteria) / total
         solution.record_reward("judge", reward, weight=self.config.score.judge_weight)
+
+
+class SharedAgenticJudgeEnv(AgenticJudgeEnv):
+    """Judge the solver in its runtime, preserving the complete mutable workspace."""
+
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        async with agents.solver.provision(task) as box:
+            solution = await agents.solver.run(task, runtime=box)
+            judge_task = JudgeTask.from_trace(solution, self.config.task)
+            await agents.judge.run(judge_task, runtime=box)
+
+
+class IsolatedAgenticJudgeEnv(AgenticJudgeEnv):
+    """Judge only collected artifacts in a fresh box with the solver's policy."""
+
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        solution = await agents.solver.run(task)
+        if not solution.ok:
+            return
+        await agents.judge.run(
+            JudgeTask.from_trace(solution, self.config.task, share_runtime=False)
+        )

verifiers/v1/envs/shared_agentic_judge/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.envs.agentic_judge.env import (
+    AgenticJudgeEnvConfig,
+    SharedAgenticJudgeEnv,
+)
+
+__all__ = ["AgenticJudgeEnvConfig", "SharedAgenticJudgeEnv"]

verifiers/v1/runtimes/base.py

@@ -332,19 +332,17 @@ class Runtime(ABC):
         return url
 
     async def prepare_setup(self) -> None:
-        """Claim the runtime for trusted setup; restricted runtimes may reject reuse."""
+        """Open egress for trusted setup when reusing a restricted runtime."""
         if not self.network_restricted:
             return
         if self._setup_claimed:
-            raise SandboxError(
-                f"network-filtered {self.type} runtimes are single-rollout; "
-                "provision a fresh runtime instead of reusing this one"
-            )
+            await self.prepare_execution(None)
         self._setup_claimed = True
 
-    async def prepare_execution(self, routes: list[str]) -> None:
+    async def prepare_execution(self, routes: list[str] | None) -> None:
         """Last setup step, right before the agent starts. Restricted runtimes enforce
-        their policy here; `routes` identifies the interception and MCP endpoints."""
+        their policy here; `routes` identifies the interception and MCP endpoints.
+        None restores unrestricted egress for another trusted setup phase."""
 
     @property
     def network_restricted(self) -> bool:

verifiers/v1/runtimes/docker/__init__.py

@@ -233,11 +233,16 @@ class DockerRuntime(Runtime):
             return url.replace(host, "host.docker.internal", 1)
         return url
 
-    async def prepare_execution(self, routes: list[str]) -> None:
+    async def prepare_execution(self, routes: list[str] | None) -> None:
         """Allow the declared framework routes, then leave the proxy as the only route."""
         if not self.network_restricted:
             return
         assert self._proxy is not None
+        if routes is None:
+            self._proxy.policy = NetworkPolicy(
+                ["*"], [], [HOST_ALIAS], allow_non_global=True
+            )
+            return
         framework = [
             urlsplit(url)._replace(path="", query="", fragment="").geturl()
             for url in routes
@@ -247,6 +252,8 @@ class DockerRuntime(Runtime):
             self.config.block,
             framework,
         )
+        if self._cut:
+            return
         script = (
             "set -eu; HOST=$1; "
             "PORT=$2; "

verifiers/v1/runtimes/prime.py

@@ -181,22 +181,25 @@ class PrimeRuntime(Runtime):
         ) as e:  # provisioning failure is one rollout's problem, not the eval's
             raise SandboxError(f"prime sandbox provisioning failed: {e}") from e
 
-    async def prepare_execution(self, routes: list[str]) -> None:
+    async def prepare_execution(self, routes: list[str] | None) -> None:
         """Apply the host policy after setup and wait until the platform enforces it."""
         if not self.network_restricted:
             return
         try:
-            hosts = list(
-                dict.fromkeys(
-                    h for h in (urlsplit(route).hostname for route in routes) if h
-                )
-            )
-            if self.config.allow == ["*"]:
-                policy = {"deny": self.config.block}
+            if routes is None:
+                policy = {"allow": ["*"]}
             else:
-                entries = list(dict.fromkeys([*hosts, *self.config.allow]))
-                validate_egress_lists(entries, None)
-                policy = {"allow": entries} if entries else {"deny": ["*"]}
+                hosts = list(
+                    dict.fromkeys(
+                        h for h in (urlsplit(route).hostname for route in routes) if h
+                    )
+                )
+                if self.config.allow == ["*"]:
+                    policy = {"deny": self.config.block}
+                else:
+                    entries = list(dict.fromkeys([*hosts, *self.config.allow]))
+                    validate_egress_lists(entries, None)
+                    policy = {"allow": entries} if entries else {"deny": ["*"]}
             status = await self._client.set_network(self.info.id, **policy)
             try:
                 async with asyncio.timeout(60):
@@ -217,8 +220,8 @@ class PrimeRuntime(Runtime):
         logger.info(
             "prime: egress policy applied on sandbox %s (allow=%s block=%s)",
             self.info.id,
-            self.config.allow,
-            self.config.block,
+            policy.get("allow"),
+            policy.get("deny"),
         )
 
     async def run(self, argv: list[str], env: dict[str, str]) -> ProgramResult:

{verifiers-0.2.2.dev84.dist-info → verifiers-0.2.2.dev86.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev84
+Version: 0.2.2.dev86
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev84.dist-info → verifiers-0.2.2.dev86.dist-info}/RECORD

@@ -231,10 +231,11 @@ verifiers/v1/dialects/base.py,sha256=YZQDnasQseDXK-GHLfJLZgyHIIygS80YhD_sTjmUbfA
 verifiers/v1/dialects/chat.py,sha256=R_otqi5N0snB1QVKIYloNeLHR9kc7OmI06HYg1ir_Lw,14021
 verifiers/v1/dialects/responses.py,sha256=ismnxUikPcPUIm80FUTPBbj-Cn1e0yGAsISMEseAWRc,13679
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-verifiers/v1/envs/agentic_judge/__init__.py,sha256=6vwtMCQ_cuvuubCF5-nrE5W7gGgzE-LIPKorkM5DXdw,309
-verifiers/v1/envs/agentic_judge/env.py,sha256=0KXuTEg4nd0jprPqFgQYFxf2JLdPVbUmljbgPOT-5Bg,13810
+verifiers/v1/envs/agentic_judge/__init__.py,sha256=LfEcRHLG_siKzhzWmODDXWSLpLgfum3UFdQDK9XG0gk,325
+verifiers/v1/envs/agentic_judge/env.py,sha256=9HdI8dm6m5QTNcHhpMvu6JJ2GlqH6e6xrc0inrEEZMk,14443
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
+verifiers/v1/envs/shared_agentic_judge/__init__.py,sha256=9T4vyYAuqgZSQjqTIceghxnpauvqeW6XGhZSQDCttMA,168
 verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZnLMje6eVgyz5U,138
 verifiers/v1/envs/single_agent/env.py,sha256=lPs_a6VA7KoH1xqA3T8S32Jzb_d4zrx0_54LrqT5z1Q,1187
 verifiers/v1/envs/user_sim/__init__.py,sha256=nnT695HdqjAEouYL5wtVu6MhIp2wf5yuK-EYgxOVx0s,118
@@ -296,12 +297,12 @@ verifiers/v1/mcp/launch.py,sha256=wBpyeiOO86l1H2lVMbLVSV9Lbp90XeyXqauuK8snfEU,20
 verifiers/v1/mcp/server.py,sha256=dRBxytFgS6rv1BS6t5EgBd3xeF6U1nTyt0ArDCLk3VE,12301
 verifiers/v1/mcp/toolset.py,sha256=T8Ioiq4wphxVOTHUDf28ZoQBM2ZYEevGnGgw8TFEMiA,1002
 verifiers/v1/runtimes/__init__.py,sha256=Kwe612RjtGzrQhc19QdXc7CmZAnQLH-EdK_xqkKP3Tk,2629
-verifiers/v1/runtimes/base.py,sha256=tZRELSfZmi3l_1BsjabnNC3mEPfqNvUNevdnRvLPv6w,16257
+verifiers/v1/runtimes/base.py,sha256=0vLTBATJplxVAgXRNqbUzX4J26p88f-r7JQmWsOugxg,16180
 verifiers/v1/runtimes/limiters.py,sha256=ZjjYUFEt20yH8ggL5uU9w9U500AOB44qSUTQ7bYyh0c,2730
 verifiers/v1/runtimes/modal.py,sha256=eFiT-Ovf_RwQZbSw3Dwv2EXeHV0d6uGvPswqKYKCw5k,8826
-verifiers/v1/runtimes/prime.py,sha256=kwvPqUI8D-GvcFwYRdpTHE1hXS0vfJGPlFOLgzm4Mvs,14756
+verifiers/v1/runtimes/prime.py,sha256=gJf-Ap5N-9Ecp-GpCNlPta7Gc9C5--Bzbk3rbK-vBIQ,14901
 verifiers/v1/runtimes/subprocess.py,sha256=ZrHRlTuTyCrcSKpiPD2ElcPpe97EvVsXJ8BfwUaNXiI,5988
-verifiers/v1/runtimes/docker/__init__.py,sha256=vyEgf6XgAwjGgBicJchJvNgcDgYXrRvyQC4W52-23GU,14556
+verifiers/v1/runtimes/docker/__init__.py,sha256=ObGNk7Uc5I4C3sumwIuD0Z30sO3FeDxl62hElfrGyWw,14775
 verifiers/v1/runtimes/docker/egress.py,sha256=37UXgNX9gnoLxjMg4KZaJIGoApCAM6MK22eJGNoMi38,14514
 verifiers/v1/serve/__init__.py,sha256=sr_zkDzAHnOcQm-QRm8xwJBmCMFD-xhKIFPaDZvMbUs,641
 verifiers/v1/serve/client.py,sha256=CxpOwCn4ehuCXfqDJeV9zj7O0XC7MPcI-OZ1L_CSDlo,7057
@@ -337,8 +338,8 @@ verifiers/v1/utils/platform.py,sha256=56Ixmk1SER6q5LvzyYcA0hgzGpzhxKZL8Hiqp4c_XV
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev84.dist-info/METADATA,sha256=hu_D0aS0L0VjJxeOvodUkjLmh1Ag4g4VU7iMaD8Yo2g,4545
-verifiers-0.2.2.dev84.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev84.dist-info/entry_points.txt,sha256=v6v0QT9vVExnfn4br42MostI_o3f3ZbWYlTormz-U3g,515
-verifiers-0.2.2.dev84.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev84.dist-info/RECORD,,
+verifiers-0.2.2.dev86.dist-info/METADATA,sha256=OcKmRg99ZpwQgZq56d-TeFt-IOtk-Jv3EUv2x_OvBcI,4545
+verifiers-0.2.2.dev86.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev86.dist-info/entry_points.txt,sha256=v6v0QT9vVExnfn4br42MostI_o3f3ZbWYlTormz-U3g,515
+verifiers-0.2.2.dev86.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev86.dist-info/RECORD,,