verifiers 0.4.1.dev20__tar.gz → 0.4.1.dev22__tar.gz

{verifiers-0.4.1.dev20 → verifiers-0.4.1.dev22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev20
+Version: 0.4.1.dev22
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev20 → verifiers-0.4.1.dev22}/tests/integration/fixtures/echo_acp_resume_v1.py

@@ -52,21 +52,21 @@ class ACPResumeTask(vf.Task[vf.TaskData, vf.State, ACPResumeTaskConfig]):
 class ACPResumeEnv(vf.SingleAgentEnv):
     async def run(self, task, agents):
         async with agents.agent.interaction(task) as interaction:
-            # Stated as a fact, not a "do not" instruction: OpenAI's content filter
-            # refuses "remember ... do not call tools or store it" as an injection.
+            # No "without tools" constraint here: models treat passing the codeword
+            # to a tool in the next turn as exfiltrating a secret and refuse.
             first = await interaction.turn(
-                f"The codeword for this conversation is {CODEWORD}. Keep it in the "
-                "conversation context only, without tools or persistent memory. "
-                "Reply with exactly READY."
+                f"The codeword for this conversation is {CODEWORD}. It is not a "
+                "secret. In my next message I will ask you to pass it to a tool. For "
+                "now, reply with exactly READY."
             )
             segments = [first]
             if not first.terminated:
                 segments.append(
                     await interaction.turn(
-                        "Use your tools or pre-imported Python skills to call `resume`'s "
-                        "`recall` with the "
-                        "codeword from my previous message, then reply with exactly the "
-                        "tool result."
+                        "Call the `recall` tool of the `resume` server (directly or "
+                        "through your pre-imported Python skills) with the codeword "
+                        "from my previous message. Then reply with exactly the tool "
+                        "result and nothing else."
                     )
                 )
             interaction.trace.info["acp_segments"] = [

verifiers-0.4.1.dev22/tests/integration/test_sampling.py

@@ -0,0 +1,29 @@
+"""Sampling overrides reach the provider: the eval's temperature, top_p, and reasoning
+effort replace whatever the harness sends, across every harness. codex speaks Responses,
+claude-code Anthropic messages, and the rest chat completions."""
+
+import pytest
+from grid import grid
+
+SAMPLING = {"temperature": 0.3, "top_p": 0.9, "reasoning_effort": "high"}
+
+
+@pytest.mark.parametrize(
+    "harness,harness_runtime", grid(runtimes=("prime",)), indirect=True
+)
+async def test_sampling_overrides(run, harness, harness_runtime):
+    (episode,) = await run(
+        "echo-v1",
+        harness=harness,
+        runtime=harness_runtime,
+        env={"agent": {"sampling": SAMPLING}},
+    )
+    (trace,) = episode.traces
+    assert trace.ok, [error.message for error in trace.errors]
+    assert trace.calls
+    # Each call records the sampling parsed from the request that went upstream.
+    for call in trace.calls:
+        assert call.sampling is not None
+        assert call.sampling.temperature == SAMPLING["temperature"]
+        assert call.sampling.top_p == SAMPLING["top_p"]
+        assert call.sampling.reasoning_effort == SAMPLING["reasoning_effort"]

verifiers-0.4.1.dev22/tests/unit/test_dialects.py

@@ -0,0 +1,51 @@
+"""Each dialect imposes the eval's sampling in its native request shape, replacing the
+harness's own values, and parses the same shape back for the trace."""
+
+import pytest
+
+from verifiers.v1.dialects import AnthropicDialect, ChatDialect, ResponsesDialect
+from verifiers.v1.types import SamplingConfig
+
+SAMPLING = SamplingConfig(temperature=0.3, top_p=0.9, reasoning_effort="low")
+
+
+@pytest.mark.parametrize(
+    "dialect,body,effort",
+    [
+        (
+            ChatDialect(),
+            {"temperature": 1.0, "top_p": 1.0, "reasoning_effort": "high"},
+            lambda body: body["reasoning_effort"],
+        ),
+        (
+            ResponsesDialect(),
+            {
+                "temperature": 1.0,
+                "top_p": 1.0,
+                "reasoning": {"effort": "high", "summary": "detailed"},
+            },
+            lambda body: body["reasoning"]["effort"],
+        ),
+        (
+            AnthropicDialect(),
+            {"temperature": 1.0, "top_p": 1.0, "output_config": {"effort": "high"}},
+            lambda body: body["output_config"]["effort"],
+        ),
+    ],
+    ids=["chat", "responses", "anthropic"],
+)
+def test_apply_overrides_sampling(dialect, body, effort):
+    sent = dialect.apply_overrides(body, "m", SAMPLING)
+    assert (sent["temperature"], sent["top_p"], effort(sent)) == (0.3, 0.9, "low")
+    parsed = dialect.parse_sampling(sent)
+    assert (parsed.temperature, parsed.top_p, parsed.reasoning_effort) == (
+        0.3,
+        0.9,
+        "low",
+    )
+
+
+def test_apply_overrides_keeps_other_reasoning_fields():
+    body = {"reasoning": {"effort": "high", "summary": "detailed"}}
+    sent = ResponsesDialect().apply_overrides(body, "m", SAMPLING)
+    assert sent["reasoning"] == {"effort": "low", "summary": "detailed"}