verifiers 0.4.1.dev21__tar.gz → 0.4.1.dev22__tar.gz

{verifiers-0.4.1.dev21 → verifiers-0.4.1.dev22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev21
+Version: 0.4.1.dev22
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.4.1.dev22/tests/integration/test_sampling.py

@@ -0,0 +1,29 @@
+"""Sampling overrides reach the provider: the eval's temperature, top_p, and reasoning
+effort replace whatever the harness sends, across every harness. codex speaks Responses,
+claude-code Anthropic messages, and the rest chat completions."""
+
+import pytest
+from grid import grid
+
+SAMPLING = {"temperature": 0.3, "top_p": 0.9, "reasoning_effort": "high"}
+
+
+@pytest.mark.parametrize(
+    "harness,harness_runtime", grid(runtimes=("prime",)), indirect=True
+)
+async def test_sampling_overrides(run, harness, harness_runtime):
+    (episode,) = await run(
+        "echo-v1",
+        harness=harness,
+        runtime=harness_runtime,
+        env={"agent": {"sampling": SAMPLING}},
+    )
+    (trace,) = episode.traces
+    assert trace.ok, [error.message for error in trace.errors]
+    assert trace.calls
+    # Each call records the sampling parsed from the request that went upstream.
+    for call in trace.calls:
+        assert call.sampling is not None
+        assert call.sampling.temperature == SAMPLING["temperature"]
+        assert call.sampling.top_p == SAMPLING["top_p"]
+        assert call.sampling.reasoning_effort == SAMPLING["reasoning_effort"]

verifiers-0.4.1.dev22/tests/unit/test_dialects.py

@@ -0,0 +1,51 @@
+"""Each dialect imposes the eval's sampling in its native request shape, replacing the
+harness's own values, and parses the same shape back for the trace."""
+
+import pytest
+
+from verifiers.v1.dialects import AnthropicDialect, ChatDialect, ResponsesDialect
+from verifiers.v1.types import SamplingConfig
+
+SAMPLING = SamplingConfig(temperature=0.3, top_p=0.9, reasoning_effort="low")
+
+
+@pytest.mark.parametrize(
+    "dialect,body,effort",
+    [
+        (
+            ChatDialect(),
+            {"temperature": 1.0, "top_p": 1.0, "reasoning_effort": "high"},
+            lambda body: body["reasoning_effort"],
+        ),
+        (
+            ResponsesDialect(),
+            {
+                "temperature": 1.0,
+                "top_p": 1.0,
+                "reasoning": {"effort": "high", "summary": "detailed"},
+            },
+            lambda body: body["reasoning"]["effort"],
+        ),
+        (
+            AnthropicDialect(),
+            {"temperature": 1.0, "top_p": 1.0, "output_config": {"effort": "high"}},
+            lambda body: body["output_config"]["effort"],
+        ),
+    ],
+    ids=["chat", "responses", "anthropic"],
+)
+def test_apply_overrides_sampling(dialect, body, effort):
+    sent = dialect.apply_overrides(body, "m", SAMPLING)
+    assert (sent["temperature"], sent["top_p"], effort(sent)) == (0.3, 0.9, "low")
+    parsed = dialect.parse_sampling(sent)
+    assert (parsed.temperature, parsed.top_p, parsed.reasoning_effort) == (
+        0.3,
+        0.9,
+        "low",
+    )
+
+
+def test_apply_overrides_keeps_other_reasoning_fields():
+    body = {"reasoning": {"effort": "high", "summary": "detailed"}}
+    sent = ResponsesDialect().apply_overrides(body, "m", SAMPLING)
+    assert sent["reasoning"] == {"effort": "low", "summary": "detailed"}