verifiers 0.2.1.dev58__tar.gz → 0.2.1.dev60__tar.gz

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev58
+Version: 0.2.1.dev60
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/tests/v1/test_e2e.py

@@ -18,6 +18,12 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns == 1
     assert trace.reward == 1.0
+    # Every sampled turn has one per-call record, linked to its assistant node.
+    sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
+    assert [c.node for c in trace.calls if c.error is None] == sampled
+    for call in trace.calls:
+        assert call.model and call.sampling is not None
+        assert call.time.duration > 0
 
 
 @pytest.mark.e2e

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/__init__.py

@@ -105,8 +105,11 @@ from verifiers.v1.trace import (
     Branch,
     Error,
     EvalRunInfo,
+    GenerationSpan,
+    ModelCall,
     RunInfo,
     TimeSpan,
+    TimeSplit,
     Timing,
     Trace,
     TraceTask,
@@ -172,6 +175,7 @@ __all__ = [
     "AgentInfo",
     "RunInfo",
     "EvalRunInfo",
+    "ModelCall",
     "TrainRunInfo",
     "VersionInfo",
     "State",
@@ -182,6 +186,8 @@ __all__ = [
     "KeptTokens",
     "Timing",
     "TimeSpan",
+    "TimeSplit",
+    "GenerationSpan",
     "Error",
     # decorators
     "stop",

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/cli/dashboard/eval.py

@@ -300,6 +300,7 @@ def _breakdown(done: list[Trace]) -> Table | None:
     have_cost = have_cached = have_reasoning = have_judge = False
     phase_secs: dict[str, float] = {}
     phase_count: dict[str, int] = {}
+    model_secs = harness_secs = 0.0
     for trace in done:
         prompt, completion, cached, reasoning, _ = _tokens(trace)
         total_in += prompt
@@ -326,6 +327,8 @@ def _breakdown(done: list[Trace]) -> Table | None:
             if span.end:  # phase was timed for this rollout
                 phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
                 phase_count[phase] = phase_count.get(phase, 0) + 1
+        model_secs += trace.timing.generation.model.duration
+        harness_secs += trace.timing.generation.harness.duration
     if (
         total_in
         or total_out
@@ -353,11 +356,18 @@ def _breakdown(done: list[Trace]) -> Table | None:
                 cost += f" + {format_cost_usd(total_judge_cost)} judge"
             usage.append(cost)
         grid.add_row("usage", "  ·  ".join(usage))
-    time_segments = [
-        f"{phase} {format_time(phase_secs[phase] / phase_count[phase])}"
-        for phase in ("boot", "setup", "generation", "finalize", "scoring")
-        if phase_count.get(phase)
-    ]
+    time_segments = []
+    for phase in ("boot", "setup", "generation", "finalize", "scoring"):
+        count = phase_count.get(phase)
+        if not count:
+            continue
+        segment = f"{phase} {format_time(phase_secs[phase] / count)}"
+        if phase == "generation":
+            segment += (
+                f" (model {format_time(model_secs / count)}"
+                f" + harness {format_time(harness_secs / count)})"
+            )
+        time_segments.append(segment)
     if time_segments:
         grid.add_row("time", "  ·  ".join(time_segments))
     return grid if grid.row_count else None

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/cli/debug.py

@@ -243,6 +243,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     except Exception as e:
         record_debug_error(trace, debug, e, setup_timeout, config.timeout.total)
     finally:
+        trace.split_generation()
         trace.info["debug"] = debug
         try:
             await runtime.stop()

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/dialects/anthropic.py

@@ -11,6 +11,7 @@ from collections.abc import Callable, Mapping
 from dataclasses import dataclass, field
 
 from anthropic.types import Message as AnthropicMessage
+from anthropic.types import Usage as AnthropicUsage
 
 from verifiers.v1.dialects.base import Dialect, StreamParser, parse_sse_event
 from verifiers.v1.types import (
@@ -21,6 +22,7 @@ from verifiers.v1.types import (
     ImageUrlSource,
     Messages,
     Response,
+    Sampling,
     SamplingConfig,
     SystemMessage,
     TextContentPart,
@@ -242,11 +244,35 @@ class AnthropicStreamParser(StreamParser):
         return response_from_wire(self.validate_response(self.message))
 
 
+class ModdedUsage(AnthropicUsage):
+    """The SDK closes `service_tier` to a fixed Literal, but Anthropic-compatible gateways
+    report their own tiers (e.g. Prime's `provisioned`). Widen to a plain string — we don't
+    consume it — so parsing stays lenient about the label instead of dropping it."""
+
+    service_tier: str | None = None  # type: ignore[assignment]
+
+
+class ModdedAnthropicMessage(AnthropicMessage):
+    usage: ModdedUsage  # type: ignore[assignment]
+
+
 class AnthropicDialect(Dialect[dict, AnthropicMessage]):
+    sampling_fields = frozenset(
+        {
+            "temperature",
+            "top_p",
+            "top_k",
+            "max_tokens",
+            "stop_sequences",
+            "thinking",
+            "tool_choice",
+            "output_config",
+        }
+    )
     routes = ("/v1/messages",)
     aux_routes = ("/v1/messages/count_tokens",)
     upstream_path = "/v1/messages"
-    response_type = AnthropicMessage
+    response_type = ModdedAnthropicMessage
 
     def auth_headers(self, api_key: str) -> dict[str, str]:
         return {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
@@ -276,17 +302,23 @@ class AnthropicDialect(Dialect[dict, AnthropicMessage]):
     def parse_response(self, response: AnthropicMessage) -> Response:
         return response_from_wire(response)
 
-    def validate_response(self, raw: dict) -> AnthropicMessage:
-        usage = raw.get("usage")
-        tier = usage.get("service_tier") if usage else None
-        if tier not in (None, "standard", "priority", "batch"):
-            raw = {**raw, "usage": usage.copy()}
-            raw["usage"].pop("service_tier")
-        return super().validate_response(raw)
-
     def stream_parser(self) -> StreamParser:
         return AnthropicStreamParser(self.validate_response)
 
+    def parse_sampling(self, body: dict) -> Sampling:
+        settings = {k: v for k, v in body.items() if k in self.sampling_fields}
+        # Lift `output_config.effort` (where `apply_overrides` puts the eval's
+        # reasoning effort) onto the typed knob; keep any other output-config keys.
+        if isinstance(config := settings.get("output_config"), dict):
+            config = dict(config)
+            if config.get("effort"):
+                settings["reasoning_effort"] = config.pop("effort")
+            if config:
+                settings["output_config"] = config
+            else:
+                settings.pop("output_config")
+        return Sampling.model_validate(settings)
+
     def apply_overrides(self, body: dict, model: str, sampling: SamplingConfig) -> dict:
         # Preserve native fields except the eval's model + sampling. `temperature`/`top_p` are
         # authoritative (always dropped, the eval's applied if set); `max_tokens` is required by

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/dialects/base.py

@@ -21,7 +21,7 @@ from typing import ClassVar, Generic, TypeVar
 from pydantic import BaseModel
 from pydantic_core import from_json
 
-from verifiers.v1.types import Messages, Response, SamplingConfig, Tool
+from verifiers.v1.types import Messages, Response, Sampling, SamplingConfig, Tool
 
 ReqT = TypeVar("ReqT")
 RespT = TypeVar("RespT", bound=BaseModel)
@@ -121,6 +121,12 @@ class Dialect(ABC, Generic[ReqT, RespT]):
     `dialects.DIALECTS` and a harness speaking that format works end-to-end (the eval client and
     interception server are generic over this interface)."""
 
+    sampling_fields: ClassVar[frozenset[str]] = frozenset()
+    """Request keys that are call settings — what shapes generation given the same
+    conversation: decoding knobs, budgets/stops, reasoning effort, output contract.
+    A whitelist, so payload, conversation state, and tracking fields can never leak
+    into the per-call record by omission; an unlisted knob is simply not recorded."""
+
     routes: ClassVar[tuple[str, ...]]
     """The endpoint path(s) a program's SDK posts model turns to. The interception server serves
     one handler per route, so the wire format is resolved from the route the SDK chose (it
@@ -167,6 +173,14 @@ class Dialect(ABC, Generic[ReqT, RespT]):
     def parse_request(self, body: ReqT) -> tuple[Messages, list[Tool] | None]:
         """The native request -> vf prompt + tools (for the trace)."""
 
+    def parse_sampling(self, body: ReqT) -> Sampling:
+        """The native request's call settings -> the canonical `Sampling` (for the
+        trace's per-call records): the `sampling_fields` whitelist, with this format's
+        aliases mapped onto the typed knobs; dialect-specific keys ride as extras."""
+        return Sampling.model_validate(
+            {k: v for k, v in body.items() if k in self.sampling_fields}
+        )
+
     @abstractmethod
     def parse_response(self, response: RespT) -> Response:
         """A native (non-streamed) response -> the vf `Response` we consume."""

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/dialects/chat.py

@@ -20,6 +20,7 @@ from verifiers.v1.types import (
     Message,
     Messages,
     Response,
+    Sampling,
     SamplingConfig,
     SystemMessage,
     Tool,
@@ -271,27 +272,47 @@ class ChatStreamParser(StreamParser):
         if self.reasoning_details:
             self.message["reasoning_details"] = self.reasoning_details
         head = self.head or {}
-        return response_from_wire(
-            ModdedChatCompletion.model_validate(
+        completion = {
+            "id": head.get("id", "vf-intercept"),
+            "object": "chat.completion",
+            "created": head.get("created", int(time.time())),
+            "model": head.get("model", ""),
+            "choices": [
                 {
-                    "id": head.get("id", "vf-intercept"),
-                    "object": "chat.completion",
-                    "created": head.get("created", int(time.time())),
-                    "model": head.get("model", ""),
-                    "choices": [
-                        {
-                            "index": 0,
-                            "message": self.message,
-                            "finish_reason": self.finish_reason or "stop",
-                        }
-                    ],
-                    "usage": self.usage,
+                    "index": 0,
+                    "message": self.message,
+                    "finish_reason": self.finish_reason or "stop",
                 }
-            )
-        )
+            ],
+            "usage": self.usage,
+        }
+        return response_from_wire(ModdedChatCompletion.model_validate(completion))
 
 
 class ChatDialect(Dialect[dict, ChatCompletion]):
+    sampling_fields = frozenset(
+        {
+            "temperature",
+            "top_p",
+            "top_k",
+            "min_p",
+            "max_tokens",
+            "max_completion_tokens",
+            "reasoning_effort",
+            "seed",
+            "stop",
+            "n",
+            "logprobs",
+            "top_logprobs",
+            "logit_bias",
+            "frequency_penalty",
+            "presence_penalty",
+            "repetition_penalty",
+            "response_format",
+            "tool_choice",
+            "parallel_tool_calls",
+        }
+    )
     routes = ("/v1/chat/completions",)
     upstream_path = "/chat/completions"
     response_type = ModdedChatCompletion
@@ -311,6 +332,14 @@ class ChatDialect(Dialect[dict, ChatCompletion]):
                     tool_names[call.id] = call.name
         return messages, parse_tools(body.get("tools"))
 
+    def parse_sampling(self, body: dict) -> Sampling:
+        settings = {k: v for k, v in body.items() if k in self.sampling_fields}
+        # Canonicalize the max-tokens alias; when both ride the wire (an eval override
+        # on top of a harness's `max_completion_tokens`), the override wins.
+        if (mct := settings.pop("max_completion_tokens", None)) is not None:
+            settings.setdefault("max_tokens", mct)
+        return Sampling.model_validate(settings)
+
     def parse_response(self, response: ChatCompletion) -> Response:
         return response_from_wire(response)
 

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/dialects/responses.py

@@ -32,6 +32,7 @@ from verifiers.v1.types import (
     ImageUrlSource,
     Messages,
     Response,
+    Sampling,
     SamplingConfig,
     SystemMessage,
     TextContentPart,
@@ -277,6 +278,20 @@ class ResponsesStreamParser(StreamParser):
 
 
 class ResponsesDialect(Dialect[dict, OpenAIResponse]):
+    sampling_fields = frozenset(
+        {
+            "temperature",
+            "top_p",
+            "max_output_tokens",
+            "max_tool_calls",
+            "reasoning",
+            "text",
+            "tool_choice",
+            "parallel_tool_calls",
+            "top_logprobs",
+            "truncation",
+        }
+    )
     routes = ("/v1/responses",)
     upstream_path = "/responses"
     response_type = OpenAIResponse
@@ -286,6 +301,22 @@ class ResponsesDialect(Dialect[dict, OpenAIResponse]):
         # trailing `[DONE]`, so the turn-ending event is the final event, not the sentinel.
         return any(marker in chunk for marker in _TERMINAL_MARKERS)
 
+    def parse_sampling(self, body: dict) -> Sampling:
+        settings = {k: v for k, v in body.items() if k in self.sampling_fields}
+        # Lift `reasoning.effort` onto the typed knob; keep any other reasoning keys
+        # (e.g. `summary`) as the wire sent them.
+        if isinstance(reasoning := settings.get("reasoning"), dict):
+            reasoning = dict(reasoning)
+            if reasoning.get("effort"):
+                settings["reasoning_effort"] = reasoning.pop("effort")
+            if reasoning:
+                settings["reasoning"] = reasoning
+            else:
+                settings.pop("reasoning")
+        if "max_output_tokens" in settings:
+            settings["max_tokens"] = settings.pop("max_output_tokens")
+        return Sampling.model_validate(settings)
+
     def parse_request(self, body: dict) -> tuple[Messages, list[Tool] | None]:
         prompt: Messages = []
         if instructions := body.get("instructions"):

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/errors.py

@@ -44,7 +44,12 @@ class ProviderError(RolloutError):
 
 class OverlongPromptError(ProviderError):
     """The prompt exceeded the model's context window — a budget limit, ended as a clean
-    truncation rather than recorded as an error."""
+    truncation rather than recorded as an error. Defaults to a 400 (what the interception
+    server surfaces for it — deterministic, so an SDK never retries it); `model_error`
+    keeps the provider's real status when the failure carried one."""
+
+    def __init__(self, message: str = "", *, status_code: int = 400) -> None:
+        super().__init__(message, status_code=status_code)
 
 
 class HarnessError(RolloutError):
@@ -128,10 +133,18 @@ def model_error(
     becomes a plain `ProviderError`. `status_code` is the HTTP status surfaced to the harness (whose
     SDK then retries 5xx/429/timeout and not 4xx); derived from an SDK error when not given. Accepts
     an SDK error (the renderer) or the provider's raw error body (the httpx proxy)."""
+    from openai import APIStatusError
+
     # Some SDK errors stringify empty; fall back to the type so the message is never blank.
     text = str(e) or (type(e).__name__ if isinstance(e, BaseException) else "")
     if any(phrase in text.casefold() for phrase in _CONTEXT_LENGTH_PHRASES):
-        return OverlongPromptError(text)
+        # Keep the provider's real status when the failure carried one; else the class
+        # default (the 400 the interception server surfaces for overlong prompts).
+        if status_code is None and isinstance(e, APIStatusError):
+            status_code = e.status_code
+        return OverlongPromptError(
+            text, **({} if status_code is None else {"status_code": status_code})
+        )
     return ProviderError(
         text,
         status_code=status_code if status_code is not None else _provider_status(e),

{verifiers-0.2.1.dev58 → verifiers-0.2.1.dev60}/verifiers/v1/graph.py

@@ -31,7 +31,6 @@ from renderers.base import MultiModalData, PlaceholderRange, RenderedTokens
 
 from verifiers.v1.types import (
     AssistantMessage,
-    FinishReason,
     KeptTokens,
     Message,
     Response,
@@ -39,7 +38,6 @@ from verifiers.v1.types import (
     TextContentPart,
     Tool,
     ToolMessage,
-    Usage,
 )
 
 if TYPE_CHECKING:
@@ -104,17 +102,12 @@ class MessageNode(StrictBaseModel):
     logprobs: list[float] = Field(default_factory=list)
     """Sampling logprobs for the sampled tokens — length equals the number of True entries in
     `mask`; empty for input messages."""
-    finish_reason: FinishReason = None
-    """The response's finish reason (assistant nodes only) — kept for truncation detection."""
     multi_modal_data: SkipJsonSchema[MultiModalData | None] = None
     """The renderer items for the images this message's content introduces (pixel tensors,
     grids, hashes, placeholders) — the only carrier of the pixels from the env server to the
     trainer. `Branch.multi_modal_data` concatenates them along the path into the training
     `mm_kwargs`. Rides the wire as raw bytes (msgpack `bin`) since pydantic can't JSON the numpy;
     kept off disk by the dump-site `exclude` in prime-rl (the tensors bloat the rollout jsonl)."""
-    usage: Usage | None = None
-    """Provider-reported token usage for this message's response (assistant nodes). Preserved
-    on the wire and on disk, including cache-read tokens when the provider reports them."""
     routed_experts: SkipJsonSchema[np.ndarray | None] = None
     """This node's slice of the MoE expert-routing array — uint8 `[len(token_ids), layers,
     top_k]`, the expert ids inference selected for exactly this node's tokens. Attributed from
@@ -343,10 +336,12 @@ class PendingTurn:
             for span in tail_spans
         ]
 
-    def commit(self, response: Response, tools: list[Tool] | None = None) -> None:
-        _commit_turn(self, response)
+    def commit(self, response: Response, tools: list[Tool] | None = None) -> int:
+        """Add this turn to the graph; returns the committed assistant node's id."""
+        assistant_id = _commit_turn(self, response)
         if tools:
             self.trace.tools = tools
+        return assistant_id
 
 
 def prepare_turn(trace: Trace, prompt: list[Message]) -> PendingTurn:
@@ -486,7 +481,7 @@ def _attribute_kept_tokens(
     node.kept_tokens = KeptTokens(ids=ids.copy(), counts=counts.copy())
 
 
-def _commit_turn(turn: PendingTurn, response: Response) -> None:
+def _commit_turn(turn: PendingTurn, response: Response) -> int:
     trace = turn.trace
     prompt = turn.prompt
     tokens = response.tokens
@@ -573,8 +568,6 @@ def _commit_turn(turn: PendingTurn, response: Response) -> None:
             else [],
             # TurnTokens is discarded after commit, so transfer its logprobs without copying.
             logprobs=tokens.completion_logprobs if tokens else [],
-            finish_reason=response.finish_reason,
-            usage=response.usage,
         )
     )
     # Register the assistant so the next turn's prompt (which restates it) reuses this node.
@@ -596,6 +589,8 @@ def _commit_turn(turn: PendingTurn, response: Response) -> None:
     # completion-aligned, so only the sampled node carries them).
     _attribute_kept_tokens(trace, assistant_id, tokens.kept_tokens if tokens else None)
 
+    return assistant_id
+
 
 # --- walking the graph (views) ---------------------------------------------------------