verifiers 0.4.1.dev18__tar.gz → 0.4.1.dev20__tar.gz

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev18
+Version: 0.4.1.dev20
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/tests/integration/conftest.py

@@ -23,7 +23,10 @@ from verifiers.v1.episode import Episode
 from verifiers.v1.types import SamplingConfig
 from verifiers.v1.utils.loaders import harness_config_type, load_environment
 
-CI_MODEL = "openai/gpt-5.6-luna"
+CI_MODEL = "internal/deepseek-v4.1-flash"
+CI_REASONING_EFFORT = "low"
+# Judge config fields that keep config-plugged judges on the CI model and effort.
+CI_JUDGE = {"model": CI_MODEL, "sampling": {"reasoning_effort": CI_REASONING_EFFORT}}
 
 # Fixture tasksets (echo-v1, echo-agentic-v1, ...) live in tests/integration/fixtures,
 # added to the path via `pythonpath` in pyproject so the loader resolves them by id.
@@ -104,7 +107,8 @@ def _eval_config(
     taskset_overrides: dict | None = None,
     runtime: dict | None = None,
     env: dict | None = None,
-    reasoning_effort: str | None = None,
+    reasoning_effort: str | None = CI_REASONING_EFFORT,
+    model: str = CI_MODEL,
     client: dict | None = None,
 ) -> EvalConfig:
     """Build the smallest `EvalConfig` that still exercises the path, used by the `run`
@@ -156,7 +160,7 @@ def _eval_config(
             "max_tokens": max_tokens,
             "reasoning_effort": reasoning_effort,
         },
-        model=CI_MODEL,
+        model=model,
         **({"client": client} if client else {}),
     )
 
@@ -196,7 +200,7 @@ async def live_ctx():
     yield ModelContext(
         model=CI_MODEL,
         client=EvalClientConfig(),
-        sampling=SamplingConfig(max_tokens=2048),
+        sampling=SamplingConfig(max_tokens=2048, reasoning_effort=CI_REASONING_EFFORT),
     )
 
 

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/tests/integration/fixtures/network_policy_v1.py

@@ -68,8 +68,10 @@ class NetworkPolicyTaskset(vf.Taskset[NetworkPolicyTask, vf.TasksetConfig]):
             "allow": {"network_allow": [ALLOWED]},
         }
         prompt = (
-            f"Use your shell to run `python3 {PROBE_PATH}` once, then reply with its "
-            "output."
+            f"Your only task: run the command `python3 {PROBE_PATH}` once in your "
+            "shell, then reply with exactly what it printed. Any network policy is "
+            "already enforced, so do not inspect, configure, or change the network or "
+            "firewall, and do not run any other command."
         )
         return [
             NetworkPolicyTask(

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/tests/integration/grid.py

@@ -59,6 +59,13 @@ RUNTIMES = ["subprocess", "docker", "podman", "apptainer", "prime", "modal", "e2
 ENGINES = ("podman", "apptainer")
 HARNESS_RUNTIMES = [runtime for runtime in RUNTIMES if runtime not in ENGINES]
 
+# Harnesses that cannot run against the CI model yet.
+# TODO: drop each entry once its fix lands.
+CI_MODEL_GAPS = {
+    "kimi-code": "Prime chat completions rejects kimi-code's prompt_cache_key",
+    "openclaw": "openclaw cannot load the custom `internal/` model provider",
+}
+
 
 def agent_seats(taskset: str, env_id: str = "") -> list[str]:
     """The agent seats the env declares for `taskset`: `agent` for a plain taskset, its
@@ -164,7 +171,10 @@ def grid(
     for harness in harnesses:
         for runtime in runtimes:
             marks = [mark(harness_id(harness)), mark(runtime)]
-            if reason := unsupported(harness, runtime, needs):
+            reason = unsupported(harness, runtime, needs) or CI_MODEL_GAPS.get(
+                harness_id(harness)
+            )
+            if reason:
                 marks.append(pytest.mark.skip(reason=reason))
             cells.append(
                 pytest.param(

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/tests/integration/test_examples.py

@@ -11,14 +11,33 @@ import subprocess
 from pathlib import Path
 
 import pytest
+from conftest import CI_JUDGE, CI_MODEL, CI_REASONING_EFFORT
 from grid import agent_seats
 
 EVAL_TIMEOUT = 600  # 10 minutes for a capped eval (-n 1 -r 1)
 
 ENVIRONMENTS = Path(__file__).parent.parent.parent / "environments"
 
-# V1 tasksets that aren't part of the default CI install.
-SKIP_EVAL = {"nemo_gym_weather"}
+SKIP_EVAL = {
+    "nemo_gym_weather": "not part of the default CI install",
+    # TODO: drop once the CI model's proposer reliably finishes within the caps.
+    "proposer_solver": "the CI model's proposer reasons past the token cap",
+}
+
+# Point each bundled judge at the CI model and effort.
+JUDGE_OVERRIDES = {
+    "interception": [
+        "--env.taskset.task.judge.model",
+        CI_MODEL,
+        "--env.taskset.task.judge.sampling.reasoning-effort",
+        CI_REASONING_EFFORT,
+    ],
+    "wiki_search": [
+        "--env.taskset.task.judges",
+        # The list replaces the default entry, so restate its `question_field`.
+        json.dumps([{"id": "wiki-search", "question_field": "question", **CI_JUDGE}]),
+    ],
+}
 
 
 def v1_tasksets() -> list[str]:
@@ -35,13 +54,15 @@ def v1_tasksets() -> list[str]:
 def test_eval(taskset: str, tmp_path: Path):
     """Run one capped rollout of `taskset` and require every episode to finish ok; a
     taskset that bundles a harness uses it by default."""
-    if taskset in SKIP_EVAL:
-        pytest.skip(f"{taskset} can't run a plain-CI smoke eval")
+    if reason := SKIP_EVAL.get(taskset):
+        pytest.skip(reason)
     if os.getenv("PRIME_API_KEY"):
         model = [
-            "-m", "openai/gpt-5.6-luna",
+            "-m", CI_MODEL,
+            "--sampling.reasoning-effort", CI_REASONING_EFFORT,
             "--client.base-url", "https://api.pinference.ai/api/v1",
             "--client.api-key-var", "PRIME_API_KEY",
+            *JUDGE_OVERRIDES.get(taskset, []),
         ]  # fmt: skip
     elif os.getenv("OPENAI_API_KEY"):
         model = [

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/tests/integration/test_resume_tool.py

@@ -27,10 +27,6 @@ async def test_resume_tool(run, harness, harness_runtime):
     assert segments[0]["terminated"] is False
     assert segments[1]["terminated"] is False
     assert trace.last_reply == segments[1]["last_reply"]
-    # Kimi Code's resumed requests replay history that differs from the turns it
-    # sampled (seen on prime), so its graph forks instead of extending one branch.
-    if harness.id != "kimi-code":
-        assert trace.num_branches == 1
     assert "tool" in segments[1]["roles"]
     assert segments[1]["tool_outputs"]
     if harness.id == "pi":

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/tests/integration/test_rubric_judge.py

@@ -1,5 +1,7 @@
 """A config-plugged rubric judge scores the rollout on top of the task's own reward."""
 
+from conftest import CI_JUDGE
+
 
 async def test_rubric_judge(run, tmp_path):
     """A config-plugged rubric judge scores the rollout on top of the taskset's own reward.
@@ -17,7 +19,9 @@ async def test_rubric_judge(run, tmp_path):
         "echo-v1",
         harness="null",
         runtime={"type": "subprocess"},
-        taskset_overrides={"task": {"judges": [{"id": "rubric", "path": str(rubric)}]}},
+        taskset_overrides={
+            "task": {"judges": [{"id": "rubric", "path": str(rubric), **CI_JUDGE}]}
+        },
         max_turns=2,
     )
     (trace,) = episode.traces

{verifiers-0.4.1.dev18 → verifiers-0.4.1.dev20}/verifiers/v1/serve/client.py

@@ -39,9 +39,12 @@ from verifiers.v1.types import SamplingConfig
 logger = logging.getLogger(__name__)
 
 ResponseT = TypeVar("ResponseT", bound=BaseResponse)
+DecodedT = TypeVar("DecodedT")
 
 
 class EnvClient:
+    """Receive streamed episodes with bounded worker decoding."""
+
     def __init__(self, address: str = "tcp://127.0.0.1:5000") -> None:
         self.address = address
         self.ctx = zmq.asyncio.Context()
@@ -51,7 +54,7 @@ class EnvClient:
         self.socket.setsockopt(zmq.RCVHWM, 0)
         self.socket.connect(address)
         self._pending: dict[str, asyncio.Future[bytes]] = {}
-        self._deltas: dict[str, Callable[[bytes], None]] = {}
+        self._deltas: dict[str, Callable[[dict], None]] = {}
         # Strong refs to in-flight fire-and-forget cancels: the loop only
         # holds weak references to tasks, so an unreferenced one can be
         # garbage-collected before it ever sends
@@ -68,7 +71,7 @@ class EnvClient:
         # handler that raises is logged and skipped rather than allowed to end the loop.
         while True:
             try:
-                frames = await self.socket.recv_multipart()
+                frames = await self.socket.recv_multipart(copy=False)
             except asyncio.CancelledError:
                 break
             try:
@@ -77,16 +80,23 @@ class EnvClient:
                         f"expected [request_id, kind, data], got {len(frames)} frames - "
                         "is the env server speaking the same serve protocol?"
                     )
-                request_id_bytes, kind, data = frames
-                request_id = request_id_bytes.decode()
+                request_id_frame, kind_frame, data_frame = frames
+                request_id = request_id_frame.bytes.decode()
+                kind = kind_frame.bytes
                 if kind == b"delta":
                     on_delta = self._deltas.get(request_id)
                     if on_delta is not None:
-                        on_delta(data)
+                        # The view retains its ZMQ frame while a cancelled worker finishes.
+                        delta = await self._decode(
+                            unpack, data_frame.buffer.toreadonly()
+                        )
+                        # Cancellation may remove the handler while the worker decodes.
+                        if self._deltas.get(request_id) is on_delta:
+                            on_delta(delta)
                     continue
                 future = self._pending.pop(request_id, None)
                 if future is not None and not future.done():
-                    future.set_result(data)
+                    future.set_result(data_frame.bytes)
             except Exception:  # keep receiving for the other requests
                 logger.warning("dropping a malformed env-server frame", exc_info=True)
 
@@ -95,11 +105,11 @@ class EnvClient:
         request: BaseRequest,
         response_type: type[ResponseT],
         timeout: float | None = None,
-        on_delta: Callable[[bytes], None] | None = None,
+        on_delta: Callable[[dict], None] | None = None,
     ) -> ResponseT:
         """Send a typed request and validate the reply into `response_type`. A
         `timeout` is only used for health polling — rollouts run untimed. `on_delta`
-        receives each `delta` frame the request streams before its reply."""
+        receives each decoded `delta` the request streams before its reply."""
         self._ensure_receiver()
         request_id = uuid.uuid4().hex
         future: asyncio.Future[bytes] = asyncio.get_running_loop().create_future()
@@ -134,13 +144,10 @@ class EnvClient:
             raise RuntimeError(response.error or "env server request failed")
         return response
 
-    async def _validate_episode(self, record: dict) -> WireEpisode:
-        """Type the assembled record off the loop, one episode at a time: a long
-        trace is a lot of token spans to validate."""
+    async def _decode(self, decode: Callable[..., DecodedT], *args: object) -> DecodedT:
+        """Decode off the loop, sharing one slot with final episode validation."""
         await self._decode_slots.acquire()
-        decoding = asyncio.create_task(
-            asyncio.to_thread(WireEpisode.model_validate, record)
-        )
+        decoding = asyncio.create_task(asyncio.to_thread(decode, *args))
         # Hold the slot until the worker finishes so cancellation cannot overlap decodes.
         decoding.add_done_callback(lambda _: self._decode_slots.release())
         try:
@@ -199,8 +206,7 @@ class EnvClient:
         episode returned is assembled from the same deltas."""
         assembly = EpisodeAssembly()
 
-        def apply(data: bytes) -> None:
-            delta = unpack(data)
+        def apply(delta: dict) -> None:
             assembly.apply(delta)
             if on_delta is not None:
                 on_delta(delta)
@@ -216,8 +222,8 @@ class EnvClient:
             on_delta=apply,
         )
         assert response.head is not None
-        return await self._validate_episode(
-            assembly.finish(response.head, response.traces)
+        return await self._decode(
+            WireEpisode.model_validate, assembly.finish(response.head, response.traces)
         )
 
     async def close(self) -> None: