verifiers 0.4.1.dev19__tar.gz → 0.4.1.dev21__tar.gz

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev19
+Version: 0.4.1.dev21
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/tests/integration/conftest.py

@@ -23,7 +23,10 @@ from verifiers.v1.episode import Episode
 from verifiers.v1.types import SamplingConfig
 from verifiers.v1.utils.loaders import harness_config_type, load_environment
 
-CI_MODEL = "openai/gpt-5.6-luna"
+CI_MODEL = "internal/deepseek-v4.1-flash"
+CI_REASONING_EFFORT = "low"
+# Judge config fields that keep config-plugged judges on the CI model and effort.
+CI_JUDGE = {"model": CI_MODEL, "sampling": {"reasoning_effort": CI_REASONING_EFFORT}}
 
 # Fixture tasksets (echo-v1, echo-agentic-v1, ...) live in tests/integration/fixtures,
 # added to the path via `pythonpath` in pyproject so the loader resolves them by id.
@@ -104,7 +107,8 @@ def _eval_config(
     taskset_overrides: dict | None = None,
     runtime: dict | None = None,
     env: dict | None = None,
-    reasoning_effort: str | None = None,
+    reasoning_effort: str | None = CI_REASONING_EFFORT,
+    model: str = CI_MODEL,
     client: dict | None = None,
 ) -> EvalConfig:
     """Build the smallest `EvalConfig` that still exercises the path, used by the `run`
@@ -156,7 +160,7 @@ def _eval_config(
             "max_tokens": max_tokens,
             "reasoning_effort": reasoning_effort,
         },
-        model=CI_MODEL,
+        model=model,
         **({"client": client} if client else {}),
     )
 
@@ -196,7 +200,7 @@ async def live_ctx():
     yield ModelContext(
         model=CI_MODEL,
         client=EvalClientConfig(),
-        sampling=SamplingConfig(max_tokens=2048),
+        sampling=SamplingConfig(max_tokens=2048, reasoning_effort=CI_REASONING_EFFORT),
     )
 
 

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/tests/integration/fixtures/echo_acp_resume_v1.py

@@ -52,21 +52,21 @@ class ACPResumeTask(vf.Task[vf.TaskData, vf.State, ACPResumeTaskConfig]):
 class ACPResumeEnv(vf.SingleAgentEnv):
     async def run(self, task, agents):
         async with agents.agent.interaction(task) as interaction:
-            # Stated as a fact, not a "do not" instruction: OpenAI's content filter
-            # refuses "remember ... do not call tools or store it" as an injection.
+            # No "without tools" constraint here: models treat passing the codeword
+            # to a tool in the next turn as exfiltrating a secret and refuse.
             first = await interaction.turn(
-                f"The codeword for this conversation is {CODEWORD}. Keep it in the "
-                "conversation context only, without tools or persistent memory. "
-                "Reply with exactly READY."
+                f"The codeword for this conversation is {CODEWORD}. It is not a "
+                "secret. In my next message I will ask you to pass it to a tool. For "
+                "now, reply with exactly READY."
             )
             segments = [first]
             if not first.terminated:
                 segments.append(
                     await interaction.turn(
-                        "Use your tools or pre-imported Python skills to call `resume`'s "
-                        "`recall` with the "
-                        "codeword from my previous message, then reply with exactly the "
-                        "tool result."
+                        "Call the `recall` tool of the `resume` server (directly or "
+                        "through your pre-imported Python skills) with the codeword "
+                        "from my previous message. Then reply with exactly the tool "
+                        "result and nothing else."
                     )
                 )
             interaction.trace.info["acp_segments"] = [

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/tests/integration/fixtures/network_policy_v1.py

@@ -68,8 +68,10 @@ class NetworkPolicyTaskset(vf.Taskset[NetworkPolicyTask, vf.TasksetConfig]):
             "allow": {"network_allow": [ALLOWED]},
         }
         prompt = (
-            f"Use your shell to run `python3 {PROBE_PATH}` once, then reply with its "
-            "output."
+            f"Your only task: run the command `python3 {PROBE_PATH}` once in your "
+            "shell, then reply with exactly what it printed. Any network policy is "
+            "already enforced, so do not inspect, configure, or change the network or "
+            "firewall, and do not run any other command."
         )
         return [
             NetworkPolicyTask(

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/tests/integration/grid.py

@@ -59,6 +59,13 @@ RUNTIMES = ["subprocess", "docker", "podman", "apptainer", "prime", "modal", "e2
 ENGINES = ("podman", "apptainer")
 HARNESS_RUNTIMES = [runtime for runtime in RUNTIMES if runtime not in ENGINES]
 
+# Harnesses that cannot run against the CI model yet.
+# TODO: drop each entry once its fix lands.
+CI_MODEL_GAPS = {
+    "kimi-code": "Prime chat completions rejects kimi-code's prompt_cache_key",
+    "openclaw": "openclaw cannot load the custom `internal/` model provider",
+}
+
 
 def agent_seats(taskset: str, env_id: str = "") -> list[str]:
     """The agent seats the env declares for `taskset`: `agent` for a plain taskset, its
@@ -164,7 +171,10 @@ def grid(
     for harness in harnesses:
         for runtime in runtimes:
             marks = [mark(harness_id(harness)), mark(runtime)]
-            if reason := unsupported(harness, runtime, needs):
+            reason = unsupported(harness, runtime, needs) or CI_MODEL_GAPS.get(
+                harness_id(harness)
+            )
+            if reason:
                 marks.append(pytest.mark.skip(reason=reason))
             cells.append(
                 pytest.param(

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/tests/integration/test_examples.py

@@ -11,14 +11,33 @@ import subprocess
 from pathlib import Path
 
 import pytest
+from conftest import CI_JUDGE, CI_MODEL, CI_REASONING_EFFORT
 from grid import agent_seats
 
 EVAL_TIMEOUT = 600  # 10 minutes for a capped eval (-n 1 -r 1)
 
 ENVIRONMENTS = Path(__file__).parent.parent.parent / "environments"
 
-# V1 tasksets that aren't part of the default CI install.
-SKIP_EVAL = {"nemo_gym_weather"}
+SKIP_EVAL = {
+    "nemo_gym_weather": "not part of the default CI install",
+    # TODO: drop once the CI model's proposer reliably finishes within the caps.
+    "proposer_solver": "the CI model's proposer reasons past the token cap",
+}
+
+# Point each bundled judge at the CI model and effort.
+JUDGE_OVERRIDES = {
+    "interception": [
+        "--env.taskset.task.judge.model",
+        CI_MODEL,
+        "--env.taskset.task.judge.sampling.reasoning-effort",
+        CI_REASONING_EFFORT,
+    ],
+    "wiki_search": [
+        "--env.taskset.task.judges",
+        # The list replaces the default entry, so restate its `question_field`.
+        json.dumps([{"id": "wiki-search", "question_field": "question", **CI_JUDGE}]),
+    ],
+}
 
 
 def v1_tasksets() -> list[str]:
@@ -35,13 +54,15 @@ def v1_tasksets() -> list[str]:
 def test_eval(taskset: str, tmp_path: Path):
     """Run one capped rollout of `taskset` and require every episode to finish ok; a
     taskset that bundles a harness uses it by default."""
-    if taskset in SKIP_EVAL:
-        pytest.skip(f"{taskset} can't run a plain-CI smoke eval")
+    if reason := SKIP_EVAL.get(taskset):
+        pytest.skip(reason)
     if os.getenv("PRIME_API_KEY"):
         model = [
-            "-m", "openai/gpt-5.6-luna",
+            "-m", CI_MODEL,
+            "--sampling.reasoning-effort", CI_REASONING_EFFORT,
             "--client.base-url", "https://api.pinference.ai/api/v1",
             "--client.api-key-var", "PRIME_API_KEY",
+            *JUDGE_OVERRIDES.get(taskset, []),
         ]  # fmt: skip
     elif os.getenv("OPENAI_API_KEY"):
         model = [

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/tests/integration/test_resume_tool.py

@@ -27,10 +27,6 @@ async def test_resume_tool(run, harness, harness_runtime):
     assert segments[0]["terminated"] is False
     assert segments[1]["terminated"] is False
     assert trace.last_reply == segments[1]["last_reply"]
-    # Kimi Code's resumed requests replay history that differs from the turns it
-    # sampled (seen on prime), so its graph forks instead of extending one branch.
-    if harness.id != "kimi-code":
-        assert trace.num_branches == 1
     assert "tool" in segments[1]["roles"]
     assert segments[1]["tool_outputs"]
     if harness.id == "pi":

{verifiers-0.4.1.dev19 → verifiers-0.4.1.dev21}/tests/integration/test_rubric_judge.py

@@ -1,5 +1,7 @@
 """A config-plugged rubric judge scores the rollout on top of the task's own reward."""
 
+from conftest import CI_JUDGE
+
 
 async def test_rubric_judge(run, tmp_path):
     """A config-plugged rubric judge scores the rollout on top of the taskset's own reward.
@@ -17,7 +19,9 @@ async def test_rubric_judge(run, tmp_path):
         "echo-v1",
         harness="null",
         runtime={"type": "subprocess"},
-        taskset_overrides={"task": {"judges": [{"id": "rubric", "path": str(rubric)}]}},
+        taskset_overrides={
+            "task": {"judges": [{"id": "rubric", "path": str(rubric), **CI_JUDGE}]}
+        },
         max_turns=2,
     )
     (trace,) = episode.traces