verifiers 0.4.1.dev22__tar.gz → 0.4.1.dev24__tar.gz

{verifiers-0.4.1.dev22 → verifiers-0.4.1.dev24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev22
+Version: 0.4.1.dev24
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev22 → verifiers-0.4.1.dev24}/tests/integration/conftest.py

@@ -27,6 +27,7 @@ CI_MODEL = "internal/deepseek-v4.1-flash"
 CI_REASONING_EFFORT = "low"
 # Judge config fields that keep config-plugged judges on the CI model and effort.
 CI_JUDGE = {"model": CI_MODEL, "sampling": {"reasoning_effort": CI_REASONING_EFFORT}}
+CI_DECISION_JUDGE = {"model": "cloudflare/clef"}
 
 # Fixture tasksets (echo-v1, echo-agentic-v1, ...) live in tests/integration/fixtures,
 # added to the path via `pythonpath` in pyproject so the loader resolves them by id.

{verifiers-0.4.1.dev22 → verifiers-0.4.1.dev24}/tests/integration/test_examples.py

@@ -1,7 +1,7 @@
 """Smoke-eval every example taskset in `environments/` through the `eval` CLI.
 
-Each taskset runs with its required harness for one short, capped rollout, so a broken
-example taskset fails CI. `compact` is excluded (it's a harness, not a taskset);
+Each taskset runs with its required harness for capped rollouts, so a broken
+example taskset fails CI. Interception exercises all four of its behaviors. `compact` is excluded (it's a harness, not a taskset);
 SWE/container tasksets need a docker/prime runtime and are covered by the agentic tests.
 """
 
@@ -11,7 +11,7 @@ import subprocess
 from pathlib import Path
 
 import pytest
-from conftest import CI_JUDGE, CI_MODEL, CI_REASONING_EFFORT
+from conftest import CI_DECISION_JUDGE, CI_MODEL, CI_REASONING_EFFORT
 from grid import agent_seats
 
 EVAL_TIMEOUT = 600  # 10 minutes for a capped eval (-n 1 -r 1)
@@ -24,18 +24,27 @@ SKIP_EVAL = {
     "proposer_solver": "the CI model's proposer reasons past the token cap",
 }
 
-# Point each bundled judge at the CI model and effort.
+# CI explicitly selects the existing decision judge for grading and interception.
 JUDGE_OVERRIDES = {
     "interception": [
+        "--env.taskset.task.judge.id",
+        "decision",
         "--env.taskset.task.judge.model",
-        CI_MODEL,
-        "--env.taskset.task.judge.sampling.reasoning-effort",
-        CI_REASONING_EFFORT,
+        CI_DECISION_JUDGE["model"],
     ],
     "wiki_search": [
         "--env.taskset.task.judges",
-        # The list replaces the default entry, so restate its `question_field`.
-        json.dumps([{"id": "wiki-search", "question_field": "question", **CI_JUDGE}]),
+        # Replace the reference judge with the existing decision judge in CI.
+        json.dumps(
+            [
+                {
+                    "id": "decision",
+                    "question_field": "question",
+                    "answer_field": "answer",
+                    **CI_DECISION_JUDGE,
+                }
+            ]
+        ),
     ],
 }
 
@@ -52,8 +61,7 @@ def v1_tasksets() -> list[str]:
 
 @pytest.mark.parametrize("taskset", v1_tasksets())
 def test_eval(taskset: str, tmp_path: Path):
-    """Run one capped rollout of `taskset` and require every episode to finish ok; a
-    taskset that bundles a harness uses it by default."""
+    """Smoke-eval each example, including all four interception behaviors."""
     if reason := SKIP_EVAL.get(taskset):
         pytest.skip(reason)
     if os.getenv("PRIME_API_KEY"):
@@ -78,10 +86,11 @@ def test_eval(taskset: str, tmp_path: Path):
         for seat in agent_seats(taskset)
         for flag in (f"--env.{seat}.max-turns", "16")
     ]
+    num_tasks = "4" if taskset == "interception" else "1"
     cmd = [
         "uv", "run", "--no-sync", "vf-eval", taskset,
         *model,
-        "-n", "1", "-r", "1", *caps,
+        "-n", num_tasks, "-r", "1", *caps,
         "--sampling.max-tokens", "4096", "--no-rich", "--no-push",
         "--output-dir", str(tmp_path), "--run.dir", "run",
         # Recipe envs run their seats one at a time by default; in parallel, a
@@ -107,3 +116,17 @@ def test_eval(taskset: str, tmp_path: Path):
         for holder in (episode, *episode["traces"])
         for error in holder.get("errors") or []
     ]
+    if taskset == "interception" and os.getenv("PRIME_API_KEY"):
+        assert len(episodes) == 4
+        judged = [
+            trace
+            for episode in episodes
+            for trace in episode["traces"]
+            if trace["info"].get("judge_calls")
+        ]
+        assert len(judged) == 1
+        assert (
+            judged[0]["info"]["judge_calls"][0]["request"]["model"]
+            == CI_DECISION_JUDGE["model"]
+        )
+        assert judged[0]["response_rewrites"]

verifiers-0.4.1.dev24/tests/integration/test_rubric_judge.py

@@ -0,0 +1,40 @@
+"""Config-plugged judges score the rollout on top of the task's own reward."""
+
+import pytest
+from conftest import CI_DECISION_JUDGE, CI_JUDGE
+
+
+@pytest.mark.parametrize("judge_id", ["rubric", "decision"])
+async def test_rubric_judge(run, tmp_path, judge_id):
+    """Keep rubric coverage and exercise Clef through the existing decision judge.
+
+    Both variants verify config loading, grading, trace records and usage; the
+    rubric variant also checks its per-criterion metric.
+    """
+    rubric = tmp_path / "rubric.toml"
+    rubric.write_text(
+        "[[criteria]]\n"
+        'name = "always_yes"\n'
+        'text = "Always satisfied — answer yes regardless of the response."\n'
+    )
+    judge = {
+        "id": judge_id,
+        **(CI_DECISION_JUDGE if judge_id == "decision" else CI_JUDGE),
+    }
+    if judge_id == "rubric":
+        judge["path"] = str(rubric)
+    (episode,) = await run(
+        "echo-v1",
+        harness="null",
+        runtime={"type": "subprocess"},
+        taskset_overrides={"task": {"judges": [judge]}},
+        max_turns=2,
+    )
+    (trace,) = episode.traces
+    assert trace.ok
+    assert trace.rewards[judge_id].score > 0  # the judge's verdict landed in the reward
+    if judge_id == "rubric":
+        assert trace.metrics["rubric/always_yes"] == 1.0
+    assert trace.info["judge_calls"]  # the call was recorded onto the trace
+    assert trace.info["judge_calls"][0]["request"]["model"] == judge["model"]
+    assert trace.extra_usage[-1].prompt_tokens > 0

{verifiers-0.4.1.dev22 → verifiers-0.4.1.dev24}/verifiers/v1/rollout.py

@@ -311,7 +311,8 @@ class Rollout:
                         self._grading_config = verifier_runtime(
                             self._grader,
                             runtime,
-                            self.deployment.verifier_config(),
+                            # Solver task requirements must not override the verifier's.
+                            self.trace.agent.config.runtime,
                             self._scoring_policy,
                         )
                     elif self._scoring_policy.runtime is not None:

{verifiers-0.4.1.dev22 → verifiers-0.4.1.dev24}/verifiers/v1/runtimes/deployment.py

@@ -55,9 +55,6 @@ class _DeploymentStrategy:
     async def prepare_verifier(self, deployment: Deployment) -> None:
         return
 
-    def verifier_config(self, deployment: Deployment) -> RuntimeConfig:
-        return deployment.backend.config
-
     @asynccontextmanager
     async def verifier(
         self, deployment: Deployment, config: RuntimeConfig, env: dict[str, str]
@@ -178,9 +175,6 @@ class Deployment:
         """
         await run_shielded(self._strategy.prepare_verifier(self))
 
-    def verifier_config(self) -> RuntimeConfig:
-        return self._strategy.verifier_config(self)
-
     async def stop_execution(self) -> None:
         await self._strategy.stop_execution(self)
 

verifiers-0.4.1.dev22/tests/integration/test_rubric_judge.py

@@ -1,31 +0,0 @@
-"""A config-plugged rubric judge scores the rollout on top of the task's own reward."""
-
-from conftest import CI_JUDGE
-
-
-async def test_rubric_judge(run, tmp_path):
-    """A config-plugged rubric judge scores the rollout on top of the taskset's own reward.
-
-    The single criterion is trivially satisfiable ("answer yes"), so any live judge model
-    scores it 1.0 — the test asserts the plumbing (config narrowing -> judge call -> reward +
-    per-criterion metric on the trace), not judge quality."""
-    rubric = tmp_path / "rubric.toml"
-    rubric.write_text(
-        "[[criteria]]\n"
-        'name = "always_yes"\n'
-        'text = "Always satisfied — answer yes regardless of the response."\n'
-    )
-    (episode,) = await run(
-        "echo-v1",
-        harness="null",
-        runtime={"type": "subprocess"},
-        taskset_overrides={
-            "task": {"judges": [{"id": "rubric", "path": str(rubric), **CI_JUDGE}]}
-        },
-        max_turns=2,
-    )
-    (trace,) = episode.traces
-    assert trace.ok
-    assert trace.rewards["rubric"].score > 0  # the judge's verdict landed in the reward
-    assert trace.metrics["rubric/always_yes"] == 1.0
-    assert trace.info["judge_calls"]  # the call was recorded onto the trace