verifiers 0.2.2.dev85__tar.gz → 0.2.2.dev86__tar.gz

{verifiers-0.2.2.dev85 → verifiers-0.2.2.dev86}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev85
+Version: 0.2.2.dev86
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev85 → verifiers-0.2.2.dev86}/tests/v1/fixtures/echo_agentic_v1.py

@@ -31,6 +31,13 @@ class EchoAgenticData(vf.TaskData):
 
 
 class EchoAgenticTask(vf.Task[EchoAgenticData]):
+    async def finalize(self, trace: vf.Trace, runtime: Runtime) -> None:
+        # Subprocess uses a runtime-owned temporary cwd, not a restorable absolute
+        # path; isolated agentic judging requires a container anyway.
+        if runtime.type == "subprocess":
+            return
+        trace.state.artifacts = await vf.collect(runtime, self.data.artifacts)
+
     @vf.reward(weight=1.0)
     async def wrote_phrase(self, runtime: Runtime) -> float:
         try:
@@ -57,6 +64,7 @@ class EchoAgenticTaskset(vf.Taskset[EchoAgenticTask, EchoAgenticConfig]):
                     ),
                     system_prompt=SYSTEM,
                     answer=phrase,
+                    artifacts=[vf.Artifact(source=TARGET)],
                 ),
                 self.config.task,
             )

{verifiers-0.2.2.dev85 → verifiers-0.2.2.dev86}/tests/v1/test_e2e.py

@@ -418,23 +418,16 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
 
 
 @pytest.mark.e2e
-async def test_env_id_agentic_judge(run_v1, tmp_path):
-    """The agentic judge over the echo taskset (needs docker): the box is
-    provisioned once from the solver's runtime policy, the solver plays in it,
-    the judge lands in the SAME box with the graded trace uploaded,
-    investigates with real execution, and its parsed verdict
-    lands on the solver's trace under the spec's reward key. Wiring, not taste:
-    the judge followed the verdict-file contract — the grade itself is the
-    model's call. Exercises the config surface too: a policy-only prompt
-    override (the verdict contract is appended regardless) and
-    reward-composition weights."""
+async def test_env_id_shared_agentic_judge(run_v1, tmp_path):
+    """The shared agentic judge provisions one restricted solver-owned box and
+    safely reuses it for the judge's empirical verification."""
     policy = tmp_path / "judge_policy.txt"
     policy.write_text("Check EMPIRICALLY that the agent echoed the word back.")
     traces = await run_v1(
         "echo-v1",
         harness=None,
         env={
-            "id": "agentic-judge",
+            "id": "shared-agentic-judge",
             "solver": {
                 "harness": {"id": "bash"},
                 "runtime": {"type": "docker", "block": ["example.com"]},
@@ -458,6 +451,8 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     (judge,) = [t for t in traces if t.agent.name == "judge"]
     assert solver.ok and judge.ok
     assert judge.agent.trainable is False
+    assert solver.agent.runtime is not None and judge.agent.runtime is not None
+    assert solver.agent.runtime.id == judge.agent.runtime.id
     # The task's own reward keeps its raw score; the rescale lands on the weight.
     assert solver.rewards["echoed"].score == 1.0
     assert solver.rewards["echoed"].weight == 0.5
@@ -465,6 +460,52 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     assert 0.0 <= solver.rewards["judge"].score <= 1.0
 
 
+@pytest.mark.e2e
+async def test_env_id_agentic_judge(run_v1, tmp_path):
+    """The isolated agentic judge destroys the restricted solver box, transfers
+    its declared artifact, and restores it in a fresh box with the same policy."""
+    policy = tmp_path / "isolated_judge_policy.txt"
+    policy.write_text(
+        "Check EMPIRICALLY that answer.txt contains exactly the requested phrase."
+    )
+    traces = await run_v1(
+        "echo-agentic-v1",
+        harness=None,
+        env={
+            "id": "agentic-judge",
+            "solver": {
+                "harness": {"id": "bash"},
+                "runtime": {"type": "docker", "block": ["example.com"]},
+            },
+            "judge": {
+                "harness": {"id": "bash"},
+                "max_output_tokens": 8192,
+            },
+            "task": {
+                "prompt": {"path": str(policy)},
+                "hint": "Do not rely on README.md",
+            },
+            "score": {"task_weight": 0.5},
+        },
+        output_dir=tmp_path,
+        max_turns=10,
+        rollout_timeout=600,
+    )
+    assert sorted(t.agent.name for t in traces) == ["judge", "solver"]
+    (solver,) = [t for t in traces if t.agent.name == "solver"]
+    (judge,) = [t for t in traces if t.agent.name == "judge"]
+    assert solver.ok and judge.ok
+    assert judge.agent.trainable is False
+    assert solver.agent.runtime is not None and judge.agent.runtime is not None
+    assert solver.agent.runtime.id != judge.agent.runtime.id
+    assert solver.agent.runtime.type == judge.agent.runtime.type == "docker"
+    assert "/app/answer.txt" in solver.state.artifacts
+    assert solver.rewards["wrote_phrase"].score == 1.0
+    assert solver.rewards["wrote_phrase"].weight == 0.5
+    assert isinstance(judge.info.get("verdict"), dict)
+    assert 0.0 <= solver.rewards["judge"].score <= 1.0
+
+
 @pytest.mark.e2e
 async def test_env_id_user_sim(run_v1, tmp_path):
     """The user-sim env over the echo taskset: a modeled user (null harness) opens

{verifiers-0.2.2.dev85 → verifiers-0.2.2.dev86}/verifiers/v1/envs/agentic_judge/__init__.py

@@ -1,16 +1,16 @@
 from verifiers.v1.envs.agentic_judge.env import (
-    AgenticJudgeEnv,
     AgenticJudgeEnvConfig,
     Criterion,
+    IsolatedAgenticJudgeEnv,
     JudgeTaskConfig,
     ScoreConfig,
     TextFile,
 )
 
 __all__ = [
-    "AgenticJudgeEnv",
     "AgenticJudgeEnvConfig",
     "Criterion",
+    "IsolatedAgenticJudgeEnv",
     "JudgeTaskConfig",
     "ScoreConfig",
     "TextFile",

{verifiers-0.2.2.dev85 → verifiers-0.2.2.dev86}/verifiers/v1/envs/agentic_judge/env.py

@@ -1,16 +1,17 @@
-"""agentic-judge: a solver plays the task, then a judge verifies the work.
-
-A reusable env (`--env.id agentic-judge` over any taskset). The solver plays the
-task in a container provisioned from its runtime policy; the judge then grades
-rubric criteria (`[env.task]`: policy prompt, criteria file) and writes its
-verdicts to `/tmp/verdict.json`, with the solver's full trace record uploaded at
-`/tmp/trace.json`. `finalize()` validates them strictly onto the solver's trace —
-`judge/<name>` metrics plus a weighted-mean `judge` reward, composed with the
-taskset's own rewards via `[env.score]` (judge-only by default).
-
-`--env.share-runtime` controls whether the judge uses the solver's runtime. It is
-enabled by default. When disabled, the judge gets a fresh runtime containing the
-task's collected artifacts.
+"""Agentic judging: a solver plays the task, then a judge verifies the work.
+
+Two reusable envs share the grading protocol. `--env.id agentic-judge` provisions
+a fresh box from the solver's runtime policy and restores only the task's collected
+artifacts; `--env.id shared-agentic-judge` explicitly runs the judge in the
+solver's box. The judge grades rubric criteria (`[env.task]`: policy prompt,
+criteria file) and writes its verdicts to `/tmp/verdict.json`, with the solver's
+full trace record uploaded at `/tmp/trace.json`. `finalize()` validates them
+strictly onto the solver's trace — `judge/<name>` metrics plus a weighted-mean
+`judge` reward, composed with the taskset's own rewards via `[env.score]`
+(judge-only by default).
+
+The environment id selects the runtime boundary; there is no mode boolean whose
+value can disagree with the environment's security and artifact semantics.
 """
 
 import json
@@ -260,20 +261,22 @@ class AgenticJudgeEnvConfig(vf.EnvConfig):
     """The solver agent. Its runtime must be a container:
     `--env.solver.runtime.type docker|prime`."""
     judge: vf.AgentConfig = vf.AgentConfig()
-    """The judge agent. Its runtime is ignored when `share_runtime` is enabled;
-    otherwise it must be a container."""
-    share_runtime: bool = True
-    """Whether the judge grades in the solver's runtime."""
+    """The judge agent. Its runtime setting is ignored: both judging modes use the
+    solver's resolved runtime policy, either by borrowing its box or provisioning a
+    fresh equivalent one."""
     task: JudgeTaskConfig = JudgeTaskConfig()
     score: ScoreConfig = ScoreConfig()
 
 
 class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
+    """Common agentic-judge protocol; subclasses choose the runtime boundary."""
+
     def __init__(self, config: AgenticJudgeEnvConfig) -> None:
-        if config.share_runtime:
-            config.judge = config.judge.model_copy(
-                update={"runtime": config.solver.runtime}
-            )
+        # Both modes use the solver's policy. Shared judging borrows that exact box;
+        # isolated judging resolves the mirrored JudgeTask into a fresh equivalent.
+        config.judge = config.judge.model_copy(
+            update={"runtime": config.solver.runtime}
+        )
         super().__init__(config)
         self._check_agents()
         # A missing policy file or a malformed rubric fails here, not mid-episode.
@@ -304,21 +307,6 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         # The judge grades the policy; its tokens are never training data.
         agents.judge.trainable = False
 
-    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
-        if self.config.share_runtime:
-            async with agents.solver.provision(task) as box:
-                solution = await agents.solver.run(task, runtime=box)
-                judge_task = JudgeTask.from_trace(solution, self.config.task)
-                await agents.judge.run(judge_task, runtime=box)
-            return
-
-        solution = await agents.solver.run(task)
-        if not solution.ok:
-            return
-        await agents.judge.run(
-            JudgeTask.from_trace(solution, self.config.task, share_runtime=False)
-        )
-
     async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
         by_agent = {t.agent.name: t for t in episode.traces}
         if "judge" not in by_agent:
@@ -336,3 +324,25 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         total = sum(criterion.weight for criterion in criteria)
         reward = sum(c.weight * scores[c.name] for c in criteria) / total
         solution.record_reward("judge", reward, weight=self.config.score.judge_weight)
+
+
+class SharedAgenticJudgeEnv(AgenticJudgeEnv):
+    """Judge the solver in its runtime, preserving the complete mutable workspace."""
+
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        async with agents.solver.provision(task) as box:
+            solution = await agents.solver.run(task, runtime=box)
+            judge_task = JudgeTask.from_trace(solution, self.config.task)
+            await agents.judge.run(judge_task, runtime=box)
+
+
+class IsolatedAgenticJudgeEnv(AgenticJudgeEnv):
+    """Judge only collected artifacts in a fresh box with the solver's policy."""
+
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        solution = await agents.solver.run(task)
+        if not solution.ok:
+            return
+        await agents.judge.run(
+            JudgeTask.from_trace(solution, self.config.task, share_runtime=False)
+        )

verifiers-0.2.2.dev86/verifiers/v1/envs/shared_agentic_judge/__init__.py

@@ -0,0 +1,6 @@
+from verifiers.v1.envs.agentic_judge.env import (
+    AgenticJudgeEnvConfig,
+    SharedAgenticJudgeEnv,
+)
+
+__all__ = ["AgenticJudgeEnvConfig", "SharedAgenticJudgeEnv"]