verifiers 0.3.2.dev52__tar.gz → 0.3.2.dev54__tar.gz

{verifiers-0.3.2.dev52 → verifiers-0.3.2.dev54}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev52
+Version: 0.3.2.dev54
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev52 → verifiers-0.3.2.dev54}/tests/v1/test_e2e.py

@@ -5,6 +5,9 @@ combinations a test runs — every axis value at least once plus the cross-bound
 with distinct networking — instead of fanning the full cross product. prime/modal rows
 are local-only (their marks are excluded in CI)."""
 
+import subprocess
+import sys
+
 import pytest
 
 mark = pytest.mark
@@ -675,6 +678,37 @@ async def test_multi_agent_env_server(run_v1_server, tmp_path):
         assert trace.metrics["duet"] == 1.0
 
 
+# `_request` parks a cancelled run's fire-and-forget cancel in `_cancel_tasks` with a
+# `discard` done-callback. One loop turn later the sends have finished but the callbacks
+# are still queued behind us, so `close()` meets a set of finished tasks.
+CLOSE_WITH_FINISHED_CANCELS = """
+import asyncio, uuid
+from verifiers.v1.serve.client import EnvClient
+
+async def main():
+    client = EnvClient("tcp://127.0.0.1:1")
+    for _ in range(3):
+        task = asyncio.get_running_loop().create_task(client._send_cancel(uuid.uuid4().hex))
+        client._cancel_tasks.add(task)
+        task.add_done_callback(client._cancel_tasks.discard)
+    await asyncio.sleep(0)
+    assert all(task.done() for task in client._cancel_tasks) and client._cancel_tasks
+    await client.close()
+
+asyncio.run(main())
+"""
+
+
+def test_env_client_close_drains_finished_cancels():
+    """`close()` returns once every fire-and-forget cancel has run, also when all of them
+    finished before it looked: the state Ctrl-C leaves behind in served mode. A child
+    process bounds the check, since a `close()` that never yields to the loop never lets
+    an in-loop timeout fire either."""
+    subprocess.run(
+        [sys.executable, "-c", CLOSE_WITH_FINISHED_CANCELS], check=True, timeout=30
+    )
+
+
 @pytest.mark.e2e
 async def test_replay_round_trip(run_v1, tmp_path):
     """eval -> replay -> replay-the-replay. Offline re-scoring must preserve the saved

{verifiers-0.3.2.dev52 → verifiers-0.3.2.dev54}/tests/v1/test_judges.py

@@ -406,12 +406,6 @@ async def test_view_modes(fake_judge_model):
     assert "SECRET REASONING" not in fake_judge_model[1]
 
 
-def test_view_defaults():
-    # reference grades the final answer; rubric grades the process by default.
-    assert vf.ReferenceJudgeConfig().view == "last_reply"
-    assert vf.RubricJudgeConfig(path="x.toml").view == "full_trace"
-
-
 async def test_rubric_view_full_trace(tmp_path, fake_judge_model):
     # The rubric judge's default view: criteria are judged against the whole transcript.
     judge = rubric_judge(tmp_path)
@@ -440,14 +434,6 @@ async def test_config_prompt_overrides_class_template(tmp_path, fake_judge_model
 # --- reference input/verdict knobs ------------------------------------------------------------
 
 
-async def test_reference_empty_response_short_circuits(fake_judge_model):
-    # An empty reply scores 0 without paying for the (foregone) judge call.
-    trace = make_trace(reply="")
-    assert await vf.ReferenceJudge().score(trace.task.data, trace) == 0.0
-    assert fake_judge_model == []
-    assert "judge" not in trace.info
-
-
 async def test_reference_list_answer(fake_judge_model):
     # A list-valued answer field is judged as multiple acceptable answers, one per line.
     class MultiTask(vf.TaskData):
@@ -512,6 +498,7 @@ async def test_error_attribution(monkeypatch, tmp_path):
     trace = make_trace(reply="")
     await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None)
     assert trace.rewards["reference"].score == 0.0
+    assert "judge_calls" not in trace.info  # the (foregone) judge call was never made
     # judge failure: unparseable verdict -> the rollout errors, no reward recorded
     trace = make_trace()
     with pytest.raises(vf.TaskError, match="no yes/no verdict"):
@@ -582,13 +569,13 @@ def test_rubric_rejects_bad_files(tmp_path):
         ).criteria
 
 
-@pytest.mark.parametrize("weight", [float("nan"), float("inf"), float("-inf")])
-def test_judge_composition_weights_are_finite(weight):
-    with pytest.raises(ValueError, match="finite number"):
-        vf.JudgeConfig(weight=weight)
-    for field in ("task_weight", "judge_weight"):
+def test_judge_composition_weights_are_finite():
+    for weight in (float("nan"), float("inf"), float("-inf")):
         with pytest.raises(ValueError, match="finite number"):
-            ScoreConfig.model_validate({field: weight})
+            vf.JudgeConfig(weight=weight)
+        for field in ("task_weight", "judge_weight"):
+            with pytest.raises(ValueError, match="finite number"):
+                ScoreConfig.model_validate({field: weight})
 
 
 async def test_rubric_score(tmp_path, fake_judge_model):
@@ -700,7 +687,7 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
     taskset = JudgedTaskset(cfg)
     trace = make_trace()
     await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None)
-    assert trace.rewards["own"].score == 0.25  # decorated rewards still run
+    assert trace.rewards["own"] == vf.Reward(score=0.25)  # decorated rewards still run
     assert trace.rewards["reference"] == vf.Reward(
         score=1.0, weight=0.5
     )  # raw score + weight, under the id-derived name
@@ -710,9 +697,3 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
     assert (
         len(trace.info["judge_calls"]) == 2
     )  # every judge call recorded (rubric = one call)
-
-
-async def test_task_without_judges_scores_as_before():
-    trace = make_trace()
-    await JudgedTask(trace.task.data).score(trace, runtime=None)
-    assert trace.rewards == {"own": vf.Reward(score=0.25)}

{verifiers-0.3.2.dev52 → verifiers-0.3.2.dev54}/tests/v1/test_taskset.py

@@ -30,11 +30,6 @@ def idxs(tasks) -> list[int]:
     return [task.data.idx for task in tasks]
 
 
-def test_head_bounds_an_infinite_taskset() -> None:
-    tasks = list(InfiniteTaskset(vf.TasksetConfig()).head(5))
-    assert idxs(tasks) == [0, 1, 2, 3, 4]
-
-
 def test_iteration_materializes_a_finite_taskset() -> None:
     taskset = FiniteTaskset(vf.TasksetConfig())
     tasks = list(taskset)
@@ -84,7 +79,8 @@ def test_head_only_builds_what_the_run_takes() -> None:
                 built.append(i)
                 yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
 
-    list(RecordingTaskset(vf.TasksetConfig()).head(3))
+    tasks = list(RecordingTaskset(vf.TasksetConfig()).head(3))
+    assert idxs(tasks) == [0, 1, 2]
     assert built == [0, 1, 2]
 
 

{verifiers-0.3.2.dev52 → verifiers-0.3.2.dev54}/verifiers/v1/serve/client.py

@@ -187,8 +187,12 @@ class EnvClient:
             self._receiver = None
         # Let scheduled fire-and-forget cancels reach the socket first, or a
         # run cancelled right before close() leaves its server-side rollout
-        # running. Loop: awaiting one wave can schedule another
+        # running. Pop before awaiting: awaiting finished tasks never yields
+        # (since CPython 3.12 a gather of them completes eagerly), so their
+        # queued `discard` callbacks cannot be what empties the set
         while self._cancel_tasks:
-            await asyncio.gather(*list(self._cancel_tasks), return_exceptions=True)
+            task = self._cancel_tasks.pop()
+            with contextlib.suppress(asyncio.CancelledError):
+                await task
         self.socket.close()
         self.ctx.term()