verifiers 0.3.2.dev205__tar.gz → 0.3.2.dev207__tar.gz

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev205
+Version: 0.3.2.dev207
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/tests/v1/test_judges.py

@@ -242,7 +242,7 @@ async def test_reference_score(fake_judge_model):
     assert len(trace.info["judge_calls"]) == 1  # the call is recorded onto the trace
     judge_record = trace.info["judge_calls"][0]
     assert judge_record["name"] == "reference"
-    assert judge_record["request"]["model"] == "openai/gpt-5.6-luna"
+    assert judge_record["request"]["model"] == "openai/gpt-6-luna"
     assert "Capital of France?" in judge_record["request"]["messages"][0]["content"]
     assert judge_record["response"]["message"] == {
         "role": "assistant",

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/tests/v1/test_trace.py

@@ -10,6 +10,7 @@ import pytest
 
 import verifiers.v1 as vf
 from verifiers.v1.agent import Interaction
+from verifiers.v1.configs.agent import resolve_agent
 from verifiers.v1.dialects.chat import ChatDialect
 from verifiers.v1.dialects.responses import ResponsesDialect, fold_assistant
 from verifiers.v1.graph import MessageNode, prepare_turn
@@ -127,10 +128,30 @@ async def test_failed_segment_does_not_reuse_prior_root_reply():
     assert trace.last_reply == "current partial reply"
 
 
-def test_bare_trace_round_trip():
+@pytest.mark.parametrize(
+    "sampling",
+    [
+        None,
+        {},
+        {"reasoning_effort": "high"},
+        {"temperature": 0.7, "max_tokens": 128, "reasoning_effort": "high"},
+        {"temperature": None},
+        {"max_completion_tokens": None},
+        {"extra_body": {"top_k": None}},
+        {"extra_body": None},
+        {"top_k": 10, "extra_body": {"frequency_penalty": 0.5}},
+        {"extra_body": {"top_k": 20, "max_completion_tokens": None}},
+    ],
+)
+def test_bare_trace_round_trip(sampling):
+    spec = vf.AgentConfig(sampling=sampling)
+    defaults = vf.SamplingConfig(temperature=0.7, max_tokens=128, top_k=40)
+    # Resolve the live overrides before serializing the effective config on a trace.
+    resolved = resolve_agent(spec, sampling=defaults)
+    assert set(vf.SamplingConfig.model_fields) <= resolved.sampling.model_fields_set
     # The minimal trace: a base task, no nodes, no extras — dump and back into a plain Trace.
     tr = vf.Trace(
-        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        agent=vf.AgentInfo(config=resolved),
         task=vf.TraceTask(
             type="Task",
             data=vf.TaskData(idx=3, prompt="hello"),
@@ -138,13 +159,24 @@ def test_bare_trace_round_trip():
             hash="content-digest",
         ),
     )
-    rt = vf.Trace.model_validate(tr.model_dump())
-    assert rt.id == tr.id
-    assert rt.task.type == "Task"
-    assert rt.task.data.idx == 3 and rt.task.data.prompt == "hello"
-    assert rt.task.key == "dataset/example-3" and rt.task.hash == "content-digest"
-    assert rt.num_turns == 0 and rt.num_branches == 0
-    assert rt.reward == 0.0 and rt.errors == []
+    episode = vf.Episode(task=tr.task, traces=[tr], ok=True)
+    for rt in (
+        vf.Trace.model_validate(tr.model_dump()),
+        vf.Trace.model_validate_json(tr.model_dump_json(exclude_none=True)),
+        vf.Trace.model_validate(tr.to_record()),
+        vf.Episode.model_validate_json(
+            episode.model_dump_json(exclude_none=True)
+        ).traces[0],
+        vf.Episode.model_validate(episode.to_record()).traces[0],
+    ):
+        assert rt.id == tr.id
+        assert rt.task.type == "Task"
+        assert rt.task.data.idx == 3 and rt.task.data.prompt == "hello"
+        assert rt.task.key == "dataset/example-3" and rt.task.hash == "content-digest"
+        assert rt.num_turns == 0 and rt.num_branches == 0
+        assert rt.reward == 0.0 and rt.errors == []
+        assert rt.agent.config == tr.agent.config
+        assert rt.agent.config.sampling.model_dump() == resolved.sampling.model_dump()
 
 
 def test_custom_task_state_round_trip(tmp_path):

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/agent.py

@@ -21,7 +21,12 @@ from verifiers.v1.clients import (
     EvalClientConfig,
     ModelContext,
 )
-from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig, agent_config_fields
+from verifiers.v1.configs.agent import (
+    AgentConfig,
+    TimeoutConfig,
+    agent_config_fields,
+    resolve_agent,
+)
 from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.dialects import parse_message
 from verifiers.v1.harness import Harness
@@ -41,7 +46,6 @@ from verifiers.v1.trace import Trace
 from verifiers.v1.types import (
     AssistantMessage,
     Messages,
-    Sampling,
     ToolMessage,
     UserMessage,
 )
@@ -550,8 +554,7 @@ class Agent:
             config = config.model_copy(
                 update={"harness": harness_config_type("bash")(id="bash")}
             )
-        if config.sampling is None:
-            config = config.model_copy(update={"sampling": Sampling()})
+        config = resolve_agent(config)
         self.config = config
         self.harness = load_harness(config.harness)
         self.ctx = ModelContext(

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/clients/train.py

@@ -375,7 +375,7 @@ class TrainClient(Client):
         prompt_ids: list[int] | None = None
         prompt_attribution: RenderedTokens | None = None
         model = body["model"]
-        sampling_params = sampling.wire_args()
+        sampling_params = sampling.model_dump(exclude_none=True)
         chat_template_kwargs = sampling_params.pop("chat_template_kwargs", None)
         cache_salt = sampling_params.pop("cache_salt", None)
         pool = ElasticRendererPool(

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/configs/agent.py

@@ -2,7 +2,7 @@
 
 from typing import Any
 
-from pydantic import BaseModel, Field, SerializeAsAny, model_validator
+from pydantic import BaseModel, Field, SerializeAsAny, field_serializer, model_validator
 from pydantic_config import BaseConfig
 
 from verifiers.v1.clients import ClientConfig
@@ -10,7 +10,7 @@ from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig
 from verifiers.v1.configs.retries import RetryConfig
 from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
 from verifiers.v1.types import SamplingConfig
-from verifiers.v1.utils.generic import deep_merge
+from verifiers.v1.utils.generic import merge_defaults
 
 
 class TimeoutConfig(BaseConfig):
@@ -57,6 +57,12 @@ class AgentConfig(BaseConfig):
     network_notice: bool = True
     """Include the runtime's network restrictions in the agent's prompt."""
 
+    @field_serializer("sampling")
+    def _sampling_overrides(self, sampling: SamplingConfig | None) -> dict | None:
+        # An unresolved agent carries only overrides across env-server boundaries.
+        # resolve_agent fills every field before the config is recorded on a trace.
+        return sampling.model_dump(exclude_unset=True) if sampling is not None else None
+
     @model_validator(mode="before")
     @classmethod
     def _resolve_harness(cls, data):
@@ -99,9 +105,7 @@ def merge_agent_defaults(config: type[BaseModel], data: Any) -> Any:
             if isinstance(field.default, AgentConfig) and isinstance(
                 data.get(name), dict
             ):
-                data[name] = deep_merge(
-                    field.default.model_dump(exclude_none=True), data[name]
-                )
+                data[name] = merge_defaults(field.default, data[name])
     return data
 
 
@@ -116,19 +120,14 @@ def resolve_agent(
     """`spec` with what it leaves unset filled from the run's defaults; its own
     sampling values merge over the run's. The one place a seat's identity resolves,
     for configured agent roles."""
-    merged = spec.sampling if sampling is None else sampling
-    if sampling is not None and spec.sampling is not None:
-        merged = sampling.model_copy(
-            update=deep_merge(
-                sampling.model_dump(exclude_unset=True),
-                spec.sampling.model_dump(exclude_unset=True),
-            )
-        )
+    defaults = sampling or SamplingConfig()
+    overrides = (spec.sampling or SamplingConfig()).model_dump(exclude_unset=True)
+    merged = {**defaults.model_dump(), **merge_defaults(defaults, overrides)}
     return spec.model_copy(
         update={
             "harness": spec.harness if spec.harness is not None else harness,
             "model": spec.model if spec.model is not None else model,
             "client": spec.client if spec.client is not None else client,
-            "sampling": merged,
+            "sampling": SamplingConfig.model_validate(merged),
         }
     )

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/configs/judge.py

@@ -17,7 +17,7 @@ class JudgeConfig(BaseClientConfig):
     name: str = ""
     """Reward key override for a plugged judge."""
     weight: FiniteFloat = 1.0
-    model: str = "openai/gpt-5.6-luna"
+    model: str = "openai/gpt-6-luna"
     sampling: SamplingConfig = SamplingConfig()
     prompt: Path | None = None
     """File whose text overrides the judge's default prompt template."""

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/dialects/anthropic.py

@@ -660,7 +660,7 @@ class AnthropicDialect(Dialect[AnthropicMessage]):
         # Preserve native fields except the eval's model + sampling. `temperature`/`top_p` are
         # authoritative (always dropped, the eval's applied if set); `max_tokens` is required by
         # the API, so the program's is kept unless the eval sets one.
-        s = sampling.wire_args()
+        s = sampling.model_dump(exclude_none=True)
         reasoning_effort = s.pop("reasoning_effort", None)
         sampling_output_config = s.pop("output_config", None)
         overrides: dict = {**s, "model": model}

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/dialects/chat.py

@@ -586,7 +586,7 @@ class ChatDialect(Dialect[ChatCompletion]):
         # Preserve the program's native fields, overlaying only what the eval owns: the model and
         # the sampling knobs it set. The selected model is authoritative even if a permissive
         # sampling config carries an extra field named `model`.
-        overrides = sampling.wire_args()
+        overrides = sampling.model_dump(exclude_none=True)
         max_token_keys = {"max_tokens", "max_completion_tokens"}
         max_tokens_overridden = not max_token_keys.isdisjoint(overrides)
         steered = {

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/dialects/responses.py

@@ -754,7 +754,7 @@ class ResponsesDialect(Dialect[OpenAIResponse]):
     ) -> RawRequest:
         # Preserve native fields except the eval's model + sampling, mapped to the Responses shape
         # (`max_tokens` -> `max_output_tokens`); sampling is authoritative.
-        s = sampling.wire_args()
+        s = sampling.model_dump(exclude_none=True)
         max_tokens = s.pop("max_tokens", None)
         reasoning_effort = s.pop("reasoning_effort", None)
         sampling_reasoning = s.pop("reasoning", None)

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/judge.py

@@ -53,6 +53,9 @@ import re
 from collections.abc import Callable, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Generic, Literal, cast
 
+from openai.types.chat.completion_create_params import (
+    CompletionCreateParamsNonStreaming,
+)
 from pydantic import BaseModel
 from typing_extensions import TypeVar
 
@@ -62,8 +65,8 @@ from verifiers.v1.configs.judge import (
     judge_key,
 )
 from verifiers.v1.dialects.chat import message_to_wire
-from verifiers.v1.types import Messages, Usage
-from verifiers.v1.utils.generic import concrete_type
+from verifiers.v1.types import Messages, SamplingConfig, Usage
+from verifiers.v1.utils.generic import concrete_type, merge_defaults
 from verifiers.v1.utils.score import parse_judge_choice
 
 if TYPE_CHECKING:
@@ -176,8 +179,24 @@ class Judge(Generic[ParsedT, ConfigT]):
             else [message_to_wire(m) for m in messages]
         )
         kwargs: dict[str, Any] = {"model": self.config.model, "messages": wire}
-        kwargs.update(self.config.sampling.model_dump(exclude_none=True))
-        kwargs.update(sampling)
+        # Transport options are SDK arguments, not model sampling parameters.
+        request_options = {
+            key: sampling.pop(key)
+            for key in ("extra_headers", "extra_query", "timeout")
+            if key in sampling
+        }
+        params = SamplingConfig.model_validate(
+            merge_defaults(self.config.sampling, sampling)
+        ).model_dump(exclude_none=True)
+        sdk_fields = CompletionCreateParamsNonStreaming.__annotations__
+        kwargs.update(
+            {key: value for key, value in params.items() if key in sdk_fields}
+        )
+        # The SDK accepts provider-specific parameters only through extra_body.
+        kwargs["extra_body"] = {
+            key: value for key, value in params.items() if key not in sdk_fields
+        }
+        kwargs.update(request_options)
 
         response: JudgeResponse[Any] | None = None
         try:

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/types.py

@@ -3,7 +3,13 @@ from dataclasses import dataclass
 from typing import Annotated, Any, Literal
 
 import numpy as np
-from pydantic import AliasChoices, BaseModel, ConfigDict, Field
+from pydantic import (
+    AliasChoices,
+    BaseModel,
+    ConfigDict,
+    Field,
+    model_validator,
+)
 from typing_extensions import TypedDict
 
 
@@ -276,17 +282,20 @@ class SamplingConfig(BaseModel):
         None, validation_alias=AliasChoices("max_tokens", "max_completion_tokens")
     )
 
-    def wire_args(self) -> dict[str, Any]:
-        """Flatten OpenAI-style ``extra_body`` before building a provider request."""
-        args = self.model_dump(exclude_none=True)
-        extra_body = {
-            key: value
-            for key, value in (args.pop("extra_body", None) or {}).items()
-            if value is not None
-        }
-        if "max_tokens" in args:
-            extra_body.pop("max_completion_tokens", None)
-        return {**extra_body, **args}
+    @model_validator(mode="before")
+    @classmethod
+    def normalize(cls, data: Any) -> Any:
+        """Use one key per parameter before merging overrides, retaining null clears."""
+        if not isinstance(data, dict):
+            return data
+        normalized = {}
+        for layer in (data.get("extra_body") or {}, data):
+            layer = dict(layer)
+            if "max_completion_tokens" in layer:
+                layer.setdefault("max_tokens", layer.pop("max_completion_tokens"))
+            layer.pop("extra_body", None)
+            normalized.update(layer)
+        return normalized
 
 
 Sampling = SamplingConfig

{verifiers-0.3.2.dev205 → verifiers-0.3.2.dev207}/verifiers/v1/utils/generic.py

@@ -4,6 +4,8 @@ from typing import TypeVar, get_args, get_origin
 
 from pydantic import BaseModel, ValidationError
 
+from verifiers.v1.types import SamplingConfig
+
 T = TypeVar("T")
 
 
@@ -52,15 +54,21 @@ def merge_defaults(defaults: BaseModel, raw: dict | None) -> dict:
     `{}` becomes `{type = "grpo", kl = 0.1}`, `{kl = 0.5}` becomes
     `{type = "grpo", kl = 0.5}`, and `{type = "max_rl"}` stays `{type = "max_rl"}`.
     """
-    # `deep_merge` detects an `id`/`type` switch only on nested blocks, so wrap the
-    # block once to detect a switch of the block itself too.
+    raw = dict(raw or {})
+    if isinstance(defaults, SamplingConfig):
+        raw = SamplingConfig.model_validate(raw).model_dump(exclude_unset=True)
     identity = {
         k: getattr(defaults, k)
         for k in ("id", "type")
         if k in type(defaults).model_fields
     }
+    if any(key in raw and raw[key] != value for key, value in identity.items()):
+        return raw
+    for name, value in defaults:
+        if isinstance(value, BaseModel) and isinstance(raw.get(name), dict):
+            raw[name] = merge_defaults(value, raw[name])
     block = {**identity, **_dump_set(defaults)}
-    return deep_merge({"block": block}, {"block": raw or {}})["block"]
+    return deep_merge(block, raw)
 
 
 def _dump_set(config: BaseModel) -> dict: