verifiers 0.4.1.dev21__tar.gz → 0.4.1.dev23__tar.gz

{verifiers-0.4.1.dev21 → verifiers-0.4.1.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev21
+Version: 0.4.1.dev23
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev21 → verifiers-0.4.1.dev23}/tests/integration/conftest.py

@@ -27,6 +27,7 @@ CI_MODEL = "internal/deepseek-v4.1-flash"
 CI_REASONING_EFFORT = "low"
 # Judge config fields that keep config-plugged judges on the CI model and effort.
 CI_JUDGE = {"model": CI_MODEL, "sampling": {"reasoning_effort": CI_REASONING_EFFORT}}
+CI_DECISION_JUDGE = {"model": "cloudflare/clef"}
 
 # Fixture tasksets (echo-v1, echo-agentic-v1, ...) live in tests/integration/fixtures,
 # added to the path via `pythonpath` in pyproject so the loader resolves them by id.

{verifiers-0.4.1.dev21 → verifiers-0.4.1.dev23}/tests/integration/test_examples.py

@@ -1,7 +1,7 @@
 """Smoke-eval every example taskset in `environments/` through the `eval` CLI.
 
-Each taskset runs with its required harness for one short, capped rollout, so a broken
-example taskset fails CI. `compact` is excluded (it's a harness, not a taskset);
+Each taskset runs with its required harness for capped rollouts, so a broken
+example taskset fails CI. Interception exercises all four of its behaviors. `compact` is excluded (it's a harness, not a taskset);
 SWE/container tasksets need a docker/prime runtime and are covered by the agentic tests.
 """
 
@@ -11,7 +11,7 @@ import subprocess
 from pathlib import Path
 
 import pytest
-from conftest import CI_JUDGE, CI_MODEL, CI_REASONING_EFFORT
+from conftest import CI_DECISION_JUDGE, CI_MODEL, CI_REASONING_EFFORT
 from grid import agent_seats
 
 EVAL_TIMEOUT = 600  # 10 minutes for a capped eval (-n 1 -r 1)
@@ -24,18 +24,27 @@ SKIP_EVAL = {
     "proposer_solver": "the CI model's proposer reasons past the token cap",
 }
 
-# Point each bundled judge at the CI model and effort.
+# CI explicitly selects the existing decision judge for grading and interception.
 JUDGE_OVERRIDES = {
     "interception": [
+        "--env.taskset.task.judge.id",
+        "decision",
         "--env.taskset.task.judge.model",
-        CI_MODEL,
-        "--env.taskset.task.judge.sampling.reasoning-effort",
-        CI_REASONING_EFFORT,
+        CI_DECISION_JUDGE["model"],
     ],
     "wiki_search": [
         "--env.taskset.task.judges",
-        # The list replaces the default entry, so restate its `question_field`.
-        json.dumps([{"id": "wiki-search", "question_field": "question", **CI_JUDGE}]),
+        # Replace the reference judge with the existing decision judge in CI.
+        json.dumps(
+            [
+                {
+                    "id": "decision",
+                    "question_field": "question",
+                    "answer_field": "answer",
+                    **CI_DECISION_JUDGE,
+                }
+            ]
+        ),
     ],
 }
 
@@ -52,8 +61,7 @@ def v1_tasksets() -> list[str]:
 
 @pytest.mark.parametrize("taskset", v1_tasksets())
 def test_eval(taskset: str, tmp_path: Path):
-    """Run one capped rollout of `taskset` and require every episode to finish ok; a
-    taskset that bundles a harness uses it by default."""
+    """Smoke-eval each example, including all four interception behaviors."""
     if reason := SKIP_EVAL.get(taskset):
         pytest.skip(reason)
     if os.getenv("PRIME_API_KEY"):
@@ -78,10 +86,11 @@ def test_eval(taskset: str, tmp_path: Path):
         for seat in agent_seats(taskset)
         for flag in (f"--env.{seat}.max-turns", "16")
     ]
+    num_tasks = "4" if taskset == "interception" else "1"
     cmd = [
         "uv", "run", "--no-sync", "vf-eval", taskset,
         *model,
-        "-n", "1", "-r", "1", *caps,
+        "-n", num_tasks, "-r", "1", *caps,
         "--sampling.max-tokens", "4096", "--no-rich", "--no-push",
         "--output-dir", str(tmp_path), "--run.dir", "run",
         # Recipe envs run their seats one at a time by default; in parallel, a
@@ -107,3 +116,17 @@ def test_eval(taskset: str, tmp_path: Path):
         for holder in (episode, *episode["traces"])
         for error in holder.get("errors") or []
     ]
+    if taskset == "interception" and os.getenv("PRIME_API_KEY"):
+        assert len(episodes) == 4
+        judged = [
+            trace
+            for episode in episodes
+            for trace in episode["traces"]
+            if trace["info"].get("judge_calls")
+        ]
+        assert len(judged) == 1
+        assert (
+            judged[0]["info"]["judge_calls"][0]["request"]["model"]
+            == CI_DECISION_JUDGE["model"]
+        )
+        assert judged[0]["response_rewrites"]

verifiers-0.4.1.dev23/tests/integration/test_rubric_judge.py

@@ -0,0 +1,40 @@
+"""Config-plugged judges score the rollout on top of the task's own reward."""
+
+import pytest
+from conftest import CI_DECISION_JUDGE, CI_JUDGE
+
+
+@pytest.mark.parametrize("judge_id", ["rubric", "decision"])
+async def test_rubric_judge(run, tmp_path, judge_id):
+    """Keep rubric coverage and exercise Clef through the existing decision judge.
+
+    Both variants verify config loading, grading, trace records and usage; the
+    rubric variant also checks its per-criterion metric.
+    """
+    rubric = tmp_path / "rubric.toml"
+    rubric.write_text(
+        "[[criteria]]\n"
+        'name = "always_yes"\n'
+        'text = "Always satisfied — answer yes regardless of the response."\n'
+    )
+    judge = {
+        "id": judge_id,
+        **(CI_DECISION_JUDGE if judge_id == "decision" else CI_JUDGE),
+    }
+    if judge_id == "rubric":
+        judge["path"] = str(rubric)
+    (episode,) = await run(
+        "echo-v1",
+        harness="null",
+        runtime={"type": "subprocess"},
+        taskset_overrides={"task": {"judges": [judge]}},
+        max_turns=2,
+    )
+    (trace,) = episode.traces
+    assert trace.ok
+    assert trace.rewards[judge_id].score > 0  # the judge's verdict landed in the reward
+    if judge_id == "rubric":
+        assert trace.metrics["rubric/always_yes"] == 1.0
+    assert trace.info["judge_calls"]  # the call was recorded onto the trace
+    assert trace.info["judge_calls"][0]["request"]["model"] == judge["model"]
+    assert trace.extra_usage[-1].prompt_tokens > 0

verifiers-0.4.1.dev23/tests/integration/test_sampling.py

@@ -0,0 +1,29 @@
+"""Sampling overrides reach the provider: the eval's temperature, top_p, and reasoning
+effort replace whatever the harness sends, across every harness. codex speaks Responses,
+claude-code Anthropic messages, and the rest chat completions."""
+
+import pytest
+from grid import grid
+
+SAMPLING = {"temperature": 0.3, "top_p": 0.9, "reasoning_effort": "high"}
+
+
+@pytest.mark.parametrize(
+    "harness,harness_runtime", grid(runtimes=("prime",)), indirect=True
+)
+async def test_sampling_overrides(run, harness, harness_runtime):
+    (episode,) = await run(
+        "echo-v1",
+        harness=harness,
+        runtime=harness_runtime,
+        env={"agent": {"sampling": SAMPLING}},
+    )
+    (trace,) = episode.traces
+    assert trace.ok, [error.message for error in trace.errors]
+    assert trace.calls
+    # Each call records the sampling parsed from the request that went upstream.
+    for call in trace.calls:
+        assert call.sampling is not None
+        assert call.sampling.temperature == SAMPLING["temperature"]
+        assert call.sampling.top_p == SAMPLING["top_p"]
+        assert call.sampling.reasoning_effort == SAMPLING["reasoning_effort"]

verifiers-0.4.1.dev23/tests/unit/test_dialects.py

@@ -0,0 +1,51 @@
+"""Each dialect imposes the eval's sampling in its native request shape, replacing the
+harness's own values, and parses the same shape back for the trace."""
+
+import pytest
+
+from verifiers.v1.dialects import AnthropicDialect, ChatDialect, ResponsesDialect
+from verifiers.v1.types import SamplingConfig
+
+SAMPLING = SamplingConfig(temperature=0.3, top_p=0.9, reasoning_effort="low")
+
+
+@pytest.mark.parametrize(
+    "dialect,body,effort",
+    [
+        (
+            ChatDialect(),
+            {"temperature": 1.0, "top_p": 1.0, "reasoning_effort": "high"},
+            lambda body: body["reasoning_effort"],
+        ),
+        (
+            ResponsesDialect(),
+            {
+                "temperature": 1.0,
+                "top_p": 1.0,
+                "reasoning": {"effort": "high", "summary": "detailed"},
+            },
+            lambda body: body["reasoning"]["effort"],
+        ),
+        (
+            AnthropicDialect(),
+            {"temperature": 1.0, "top_p": 1.0, "output_config": {"effort": "high"}},
+            lambda body: body["output_config"]["effort"],
+        ),
+    ],
+    ids=["chat", "responses", "anthropic"],
+)
+def test_apply_overrides_sampling(dialect, body, effort):
+    sent = dialect.apply_overrides(body, "m", SAMPLING)
+    assert (sent["temperature"], sent["top_p"], effort(sent)) == (0.3, 0.9, "low")
+    parsed = dialect.parse_sampling(sent)
+    assert (parsed.temperature, parsed.top_p, parsed.reasoning_effort) == (
+        0.3,
+        0.9,
+        "low",
+    )
+
+
+def test_apply_overrides_keeps_other_reasoning_fields():
+    body = {"reasoning": {"effort": "high", "summary": "detailed"}}
+    sent = ResponsesDialect().apply_overrides(body, "m", SAMPLING)
+    assert sent["reasoning"] == {"effort": "low", "summary": "detailed"}

verifiers-0.4.1.dev21/tests/integration/test_rubric_judge.py

@@ -1,31 +0,0 @@
-"""A config-plugged rubric judge scores the rollout on top of the task's own reward."""
-
-from conftest import CI_JUDGE
-
-
-async def test_rubric_judge(run, tmp_path):
-    """A config-plugged rubric judge scores the rollout on top of the taskset's own reward.
-
-    The single criterion is trivially satisfiable ("answer yes"), so any live judge model
-    scores it 1.0 — the test asserts the plumbing (config narrowing -> judge call -> reward +
-    per-criterion metric on the trace), not judge quality."""
-    rubric = tmp_path / "rubric.toml"
-    rubric.write_text(
-        "[[criteria]]\n"
-        'name = "always_yes"\n'
-        'text = "Always satisfied — answer yes regardless of the response."\n'
-    )
-    (episode,) = await run(
-        "echo-v1",
-        harness="null",
-        runtime={"type": "subprocess"},
-        taskset_overrides={
-            "task": {"judges": [{"id": "rubric", "path": str(rubric), **CI_JUDGE}]}
-        },
-        max_turns=2,
-    )
-    (trace,) = episode.traces
-    assert trace.ok
-    assert trace.rewards["rubric"].score > 0  # the judge's verdict landed in the reward
-    assert trace.metrics["rubric/always_yes"] == 1.0
-    assert trace.info["judge_calls"]  # the call was recorded onto the trace