verifiers 0.1.15.dev396__tar.gz → 0.1.15.dev397__tar.gz

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev396
+Version: 0.1.15.dev397
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/verifiers/v1/GUIDE.md

@@ -254,6 +254,60 @@ async def verify(self, task, trace, runtime) -> float:
     return float(r.stdout.strip() == "1.0")
 ```
 
+### Judges
+
+When grading can't be deterministic, `vf.Judge` is a reusable LLM judge: it owns the OpenAI client
+(with the Prime key/team fallback), the call, and usage/cost capture, and leaves the two things that
+differ as hooks — `build_messages` (prompt) and `parse` (verdict). Subclass it, build it once in the
+taskset `__init__`, and call it from a `@reward`:
+
+```python
+import verifiers.v1 as vf
+
+class CorrectnessJudge(vf.Judge[bool]):                 # Judge[ParsedT] — ParsedT is your verdict type
+    prompt = "Question: {question}\nAnswer: {answer}\nResponse: {response}\nCorrect? Reply yes or no."
+
+    def parse(self, response: vf.JudgeResponse[bool]) -> bool:
+        return response.text.strip().lower().startswith("yes")
+
+class MyConfig(vf.TasksetConfig):
+    judge: vf.JudgeConfig = vf.JudgeConfig(model="openai/gpt-5-mini")
+
+class MyTaskset(vf.Taskset[MyTask, MyConfig]):
+    def __init__(self, config: MyConfig) -> None:
+        super().__init__(config)
+        self.judge = CorrectnessJudge(config.judge)
+
+    @vf.reward()
+    async def correct(self, task, trace) -> float:
+        result = await self.judge.evaluate(trace=trace, question=task.question, answer=task.answer, response=...)
+        return 1.0 if result.parsed else 0.0
+```
+
+`evaluate(*, trace=None, **fields)` renders the prompt (`build_messages`), calls the model, and
+parses the verdict (`parse`), returning a `JudgeResponse{text, parsed, usage}`. Passing `trace=`
+**records the call onto it** — a typed record appended to `trace.info["judge"]` (for debugging) and
+the call's tokens + cost added to `trace.extra_usage`, kept separate from the agent's `trace.usage`
+and off the message graph (so the trainer's token math is unaffected); the eval dashboard shows the
+agent's usage and `+judge` separately. The record lands even if the judge refuses, returns an empty
+structured output, or `parse` raises (the request was already billed). Omit `trace` for a pure call
+(e.g. in tests).
+
+The two hooks:
+
+| hook | default | override for |
+| --- | --- | --- |
+| `build_messages(**fields) -> str \| Messages` | formats the `prompt` template with the fields into one user message | a system+user / non-template prompt (return a `vf.Messages` list) |
+| `parse(response) -> ParsedT` | the structured object if `schema` is set, else `response.text` | your verdict (`bool`, a grade `str`, a pydantic model, a `list[float]`, …) |
+
+Good to know:
+
+- **Per-task rubric** is just a field — `prompt = "{task.rubric}\n…"` with `evaluate(trace=trace, task=task, …)` (`str.format` does attribute access on a passed-in `task`). For per-task *parsing*, parse in the reward, where the task is in scope.
+- **Structured outputs**: set `schema` to a pydantic model to use OpenAI structured outputs (where the provider supports it — most do); `JudgeResponse.parsed` is then the validated object. For an unsupported model, prompt for JSON and call `Model.model_validate_json(response.text)` in `parse`.
+- **Multiple / dynamic calls per rollout** (e.g. one per table column): call the low-level `complete(messages, *, trace=, schema=, parse=)` directly with `vf.Messages` you build — it records each call when passed `trace`.
+- **Config**: `JudgeConfig` adds `model` + `sampling` (a `JudgeSamplingConfig`) to `BaseClientConfig` (`base_url`/`api_key_var`/`headers`, Prime auto-config). CLI-overridable: `--taskset.judge.model …`, `--taskset.judge.sampling.max-tokens …`.
+- **Errors propagate**: a judge API failure errors the rollout (recorded as a `TasksetError` on the trace); the OpenAI SDK already retries transient 429/5xx/connection errors.
+
 ## Stop conditions
 
 A rollout ends when the harness finishes, a framework budget trips (`--max-turns`, token caps), or

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/verifiers/v1/__init__.py

@@ -38,6 +38,7 @@ from verifiers.v1.errors import (
     UserError,
 )
 from verifiers.v1.harness import Harness, HarnessConfig
+from verifiers.v1.judge import Judge, JudgeConfig, JudgeResponse, JudgeSamplingConfig
 from verifiers.v1.loaders import (
     default_harness_id,
     harness_config_type,
@@ -189,6 +190,11 @@ __all__ = [
     "taskset_config_type",
     "harness_config_type",
     "default_harness_id",
+    # judge
+    "Judge",
+    "JudgeConfig",
+    "JudgeSamplingConfig",
+    "JudgeResponse",
     # mcp
     "Toolset",
     "ToolsetConfig",

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/verifiers/v1/cli/dashboard/eval.py

@@ -25,6 +25,7 @@ from verifiers.v1.cli.output import output_path
 from verifiers.v1.configs.eval import EvalConfig
 from verifiers.v1.rollout import Phase, Rollout
 from verifiers.v1.trace import Trace
+from verifiers.v1.types import Usage
 from verifiers.v1.utils.format import format_count, format_mean, format_time
 from verifiers.utils.pricing_utils import format_cost_usd
 
@@ -199,8 +200,9 @@ def _breakdown(done: list[Trace]) -> Table | None:
     # cost are summed; each timing phase is averaged over the rollouts that have it timed (averaged
     # per phase rather than summed, since phases run concurrently across rollouts).
     total_in = total_out = total_cached = total_reasoning = 0
-    total_cost = 0.0
-    have_cost = have_cached = have_reasoning = False
+    total_judge_in = total_judge_out = 0
+    total_cost = total_judge_cost = 0.0
+    have_cost = have_cached = have_reasoning = have_judge = False
     phase_secs: dict[str, float] = {}
     phase_count: dict[str, int] = {}
     for trace in done:
@@ -216,12 +218,27 @@ def _breakdown(done: list[Trace]) -> Table | None:
         if trace.usage is not None and trace.usage.cost is not None:
             total_cost += trace.usage.cost
             have_cost = True
+        # Judge / auxiliary scoring calls (off the message graph) shown separately from the agent's.
+        judge = Usage.aggregate(trace.extra_usage)
+        if judge is not None:
+            total_judge_in += judge.input_tokens
+            total_judge_out += judge.completion_tokens
+            if judge.cost is not None:
+                total_judge_cost += judge.cost
+            have_judge = True
         for phase in ("setup", "generation", "finalize", "scoring"):
             span = getattr(trace.timing, phase)
             if span.end:  # phase was timed for this rollout
                 phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
                 phase_count[phase] = phase_count.get(phase, 0) + 1
-    if total_in or total_out or have_cost or have_cached or have_reasoning:
+    if (
+        total_in
+        or total_out
+        or have_cost
+        or have_cached
+        or have_reasoning
+        or have_judge
+    ):
         tokens = f"{format_count(total_in)}/{format_count(total_out)} tokens"
         details = []
         if have_cached:
@@ -231,8 +248,15 @@ def _breakdown(done: list[Trace]) -> Table | None:
         if details:
             tokens += f" ({', '.join(details)})"
         usage = [tokens]
+        if have_judge:
+            usage.append(
+                f"+ {format_count(total_judge_in)}/{format_count(total_judge_out)} judge"
+            )
         if have_cost:
-            usage.append(f"total {format_cost_usd(total_cost)}")
+            cost = format_cost_usd(total_cost)
+            if total_judge_cost:
+                cost += f" + {format_cost_usd(total_judge_cost)} judge"
+            usage.append(cost)
         grid.add_row("usage", "  ·  ".join(usage))
     time_segments = [
         f"{phase} {format_time(phase_secs[phase] / phase_count[phase])}"

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/verifiers/v1/clients/config.py

@@ -88,7 +88,9 @@ ClientConfig = Annotated[
 ]
 
 
-def resolve_client(config: BaseClientConfig) -> Client:
+def resolve_api_key(config: BaseClientConfig) -> str:
+    """The API key for `config`: its env var, falling back to the Prime CLI config for a
+    `PRIME_API_KEY`-keyed pinference endpoint. `"EMPTY"` when unset."""
     api_key = os.environ.get(config.api_key_var)
     host = urlparse(config.base_url).hostname or ""
     if (
@@ -97,19 +99,29 @@ def resolve_client(config: BaseClientConfig) -> Client:
         and (host == PRIME_INFERENCE_HOST or host.endswith(f".{PRIME_INFERENCE_HOST}"))
     ):
         api_key = load_prime_config().get("api_key")
-    api_key = api_key or "EMPTY"
+    return api_key or "EMPTY"
+
+
+def build_async_openai(config: BaseClientConfig) -> AsyncOpenAI:
+    """An `AsyncOpenAI` for `config` (resolved key + extra headers) — for in-env model calls
+    (e.g. a judge) and the training client's engine connection."""
+    return AsyncOpenAI(
+        base_url=config.base_url,
+        api_key=resolve_api_key(config),
+        default_headers=config.headers or None,
+    )
+
+
+def resolve_client(config: BaseClientConfig) -> Client:
     if isinstance(config, TrainClientConfig):
         # The renderer calls a vLLM `/inference/v1/generate` engine through the OpenAI SDK.
-        openai = AsyncOpenAI(
-            base_url=config.base_url,
-            api_key=api_key,
-            default_headers=config.headers or None,
-        )
         return TrainClient(
-            openai,
+            build_async_openai(config),
             pool_size=config.pool_size,
             config=config.renderer,
             renderer_model_name=config.renderer_model_name,
         )
     # The proxy is a raw httpx forwarder; the dialect supplies the auth scheme + upstream path.
-    return EvalClient(config.base_url, api_key, headers=config.headers or None)
+    return EvalClient(
+        config.base_url, resolve_api_key(config), headers=config.headers or None
+    )

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/verifiers/v1/dialects/chat.py

@@ -169,25 +169,7 @@ def response_from_wire(completion: ChatCompletion) -> Response:
     finish: FinishReason = (
         choice.finish_reason if choice.finish_reason in FINISH_REASONS else None
     )
-    usage = None
-    if completion.usage:
-        cached = (
-            completion.usage.prompt_tokens_details.cached_tokens
-            if completion.usage.prompt_tokens_details
-            else None
-        )
-        reasoning = (
-            completion.usage.completion_tokens_details.reasoning_tokens
-            if completion.usage.completion_tokens_details
-            else None
-        )
-        usage = Usage(
-            prompt_tokens=completion.usage.prompt_tokens - (cached or 0),
-            completion_tokens=completion.usage.completion_tokens,
-            cached_input_tokens=cached,
-            reasoning_tokens=reasoning,
-            cost=getattr(completion.usage, "cost", None),
-        )
+    usage = Usage.from_openai(completion.usage)
     return Response(
         id=completion.id,
         created=completion.created,

verifiers-0.1.15.dev397/verifiers/v1/judge.py

@@ -0,0 +1,180 @@
+"""A reusable per-task LLM judge for v1 tasksets.
+
+Most tasksets that can't grade deterministically reach for the same shape: an OpenAI-compatible
+endpoint, a prompt built from `(question, answer, response)`, one chat call, and a verdict parsed
+out of the reply. `Judge` centralizes that — the client construction (incl. the Prime key/team
+fallback), the call, and usage/cost capture — and leaves the two things that actually differ as
+hooks: `build_messages` (prompt setup) and `parse`
+(verdict parsing). Set `schema` to use OpenAI structured outputs (where the provider supports it),
+in which case `JudgeResponse.parsed` is the validated pydantic object.
+
+    class CorrectnessJudge(vf.Judge[bool]):
+        prompt = "Question: {question}\\nAnswer: {answer}\\nResponse: {response}\\nCorrect? yes/no"
+
+        def parse(self, response: vf.JudgeResponse[bool]) -> bool:
+            return response.text.strip().lower().startswith("yes")
+
+    self.judge = CorrectnessJudge(self.config.judge)  # self.config.judge: vf.JudgeConfig
+
+    @vf.reward
+    async def correct(self, task, trace) -> float:
+        result = await self.judge.evaluate(
+            trace=trace, question=task.question, answer=task.answer, response=...
+        )
+        return float(result.parsed)
+
+Passing `trace=` records the call onto it — a typed record appended to `trace.info["judge"]` and
+the call's tokens + cost added to `trace.extra_usage` (kept separate from the agent's `trace.usage`),
+so judge behaviour and spend are no longer invisible. The record lands even if the judge refuses, an
+empty structured output comes back, or `parse` raises (the request was already billed). Omit `trace`
+for a pure call (e.g. in tests).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, cast
+
+from openai import AsyncOpenAI
+from pydantic import BaseModel
+
+from verifiers.v1.clients.config import BaseClientConfig, build_async_openai
+from verifiers.v1.dialects.chat import message_to_wire
+from verifiers.v1.types import Messages, SamplingConfig, StrictBaseModel, Usage
+
+if TYPE_CHECKING:
+    from verifiers.v1.trace import Trace
+
+ParsedT = TypeVar("ParsedT")
+
+
+class JudgeSamplingConfig(SamplingConfig):
+    """Sampling knobs for a judge call (`temperature` / `top_p` / `reasoning_effort` /
+    `max_tokens`, plus provider-specific keys via `extra='allow'`). Same shape as the rollout's
+    `SamplingConfig` — set e.g. `judge.sampling.max_tokens`."""
+
+
+class JudgeConfig(BaseClientConfig):
+    """An LLM-judge endpoint. Inherits `base_url` / `api_key_var` / `headers` (with the Prime
+    auto-config) from `BaseClientConfig`; adds the model and sampling. Subclass to add
+    taskset-specific fields."""
+
+    model: str = "openai/gpt-5-mini"
+    sampling: JudgeSamplingConfig = JudgeSamplingConfig()
+
+
+class JudgeResponse(StrictBaseModel, Generic[ParsedT]):
+    """One judge call's result — returned to the caller and (JSON-serialized) appended to
+    `trace.info["judge"]` for debugging, including the provider-reported `usage` (tokens + cost)."""
+
+    text: str
+    """The judge's raw reply."""
+    parsed: ParsedT | None = None
+    """The verdict the taskset acts on (`parse`'s output, or the structured object for `schema`)."""
+    usage: Usage | None = None
+
+
+class Judge(Generic[ParsedT]):
+    """A per-task LLM judge over an OpenAI-compatible endpoint.
+
+    Override `build_messages` (prompt setup) and `parse` (verdict parsing) — or just set the
+    `prompt` template and use the defaults — then call `evaluate(**fields)`. Set `schema` to opt
+    into structured outputs.
+    """
+
+    prompt: str | None = None
+    """Default template for `build_messages`, formatted with the `evaluate` kwargs and sent as a
+    single user message. Override `build_messages` for system+user or non-template prompts."""
+    schema: type[BaseModel] | None = None
+    """Pydantic schema for OpenAI structured outputs. When set, the call uses
+    `response_format=schema` and `JudgeResponse.parsed` is the validated object (provider must
+    support structured outputs)."""
+
+    def __init__(self, config: JudgeConfig | None = None) -> None:
+        self.config = config or JudgeConfig()
+        self.client: AsyncOpenAI = build_async_openai(self.config)
+
+    def build_messages(self, **fields: Any) -> str | Messages:
+        """Prompt-setup hook: turn the `evaluate` fields into the messages to send (a single user
+        message as a plain `str`, or a `vf.Messages` list). The default formats the `prompt`
+        template with the fields; override it for a system+user / non-template prompt."""
+        if self.prompt is None:
+            raise ValueError(
+                f"{type(self).__name__} has no `prompt`; set it or override build_messages"
+            )
+        return self.prompt.format(**fields)
+
+    def parse(self, response: JudgeResponse[ParsedT]) -> ParsedT:
+        """Parsing hook: turn a `JudgeResponse` into the verdict. The default returns the
+        structured object when `schema` is set, otherwise the raw reply text."""
+        if self.schema is not None:
+            return cast(ParsedT, response.parsed)
+        return cast(ParsedT, response.text)
+
+    async def complete(
+        self,
+        messages: str | Messages,
+        *,
+        trace: "Trace | None" = None,
+        schema: type[BaseModel] | None = None,
+        parse: Callable[[JudgeResponse[Any]], Any] | None = None,
+        **sampling: Any,
+    ) -> JudgeResponse[Any]:
+        """Call the judge once: send `messages`, build the `JudgeResponse`, and — when a `trace` is
+        given — record it (`Trace.record_judge`) in a `finally`, so the call's tokens + cost are
+        captured even when a refusal / empty structured output / `parse` hook raises after the
+        billed request. `schema` opts into structured outputs; `parse` sets `JudgeResponse.parsed`."""
+        wire = (
+            [{"role": "user", "content": messages}]
+            if isinstance(messages, str)
+            else [message_to_wire(m) for m in messages]
+        )
+        kwargs: dict[str, Any] = {"model": self.config.model, "messages": wire}
+        kwargs.update(self.config.sampling.model_dump(exclude_none=True))
+        kwargs.update(sampling)
+
+        response: JudgeResponse[Any] | None = None
+        try:
+            if schema is not None:
+                completion = await self.client.beta.chat.completions.parse(
+                    response_format=schema, **kwargs
+                )
+                choice = completion.choices[0]
+                response = JudgeResponse(
+                    text=choice.message.content or "",
+                    parsed=choice.message.parsed,
+                    usage=Usage.from_openai(completion.usage),
+                )
+                if choice.message.refusal is not None:
+                    raise RuntimeError(
+                        f"judge refused structured output: {choice.message.refusal}"
+                    )
+                if response.parsed is None:
+                    # No refusal but no object either — e.g. a truncated/malformed reply. Surface it
+                    # rather than returning a None verdict callers read as a (falsy) failure.
+                    raise RuntimeError(
+                        f"judge returned no parseable structured output "
+                        f"(finish_reason={choice.finish_reason})"
+                    )
+            else:
+                completion = await self.client.chat.completions.create(**kwargs)
+                response = JudgeResponse(
+                    text=completion.choices[0].message.content or "",
+                    usage=Usage.from_openai(completion.usage),
+                )
+            if parse is not None:
+                response.parsed = parse(response)
+            return response
+        finally:
+            if trace is not None and response is not None:
+                trace.record_judge(response)
+
+    async def evaluate(
+        self, *, trace: "Trace | None" = None, **fields: Any
+    ) -> JudgeResponse[ParsedT]:
+        """Render the prompt (`build_messages(**fields)`), call the judge, and parse the verdict
+        (`parse`). When a `trace` is given the call is recorded onto it (`trace.info["judge"]` +
+        usage on `trace.extra_usage`), even if a refusal / empty output / parse raises."""
+        messages = self.build_messages(**fields)
+        return await self.complete(
+            messages, trace=trace, schema=self.schema, parse=self.parse
+        )

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/verifiers/v1/trace.py

@@ -13,12 +13,15 @@ import time
 import traceback
 import uuid
 from collections.abc import Mapping
-from typing import Any, Generic, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 import numpy as np
 from pydantic import Field, PrivateAttr
 from renderers.base import MultiModalData
 
+if TYPE_CHECKING:
+    from verifiers.v1.judge import JudgeResponse
+
 from verifiers.v1 import graph
 from verifiers.v1.errors import ProviderError
 from verifiers.v1.graph import MessageNode
@@ -258,6 +261,13 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
     / `to_record`), unlike `info` which persists. Type it via `Taskset[Task, Config, MyState]`; defaults
     to the base `State`."""
 
+    extra_usage: list[Usage] = Field(default_factory=list)
+    """Token usage from model calls that aren't part of the message graph — LLM judges and other
+    auxiliary scoring calls (see `verifiers.v1.judge`). Kept separate from `usage` (the agent's own
+    spend) and off the graph, so branch/turn counts and the trainer's token math (`total_tokens`,
+    `branches`) see only sampled nodes; consumers that want a grand total add the two (e.g. the eval
+    dashboard shows the agent's usage and `+judge` separately)."""
+
     is_completed: bool = False
     stop_condition: str | None = None
     errors: list[Error] = Field(default_factory=list)
@@ -396,6 +406,14 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
         for name, value in values.items():
             self.record_metric(name, value)
 
+    def record_judge(self, response: "JudgeResponse") -> None:
+        """Persist a judge call (`Judge.evaluate` / `Judge.complete` is pure): append the typed
+        response to `info["judge"]` for debugging and fold its tokens + cost into `extra_usage`
+        (→ `usage`)."""
+        self.info.setdefault("judge", []).append(response.model_dump())
+        if response.usage is not None:
+            self.extra_usage.append(response.usage)
+
     def record_reward(self, name: str, value: float, weight: float = 1.0) -> None:
         """Record a `@reward`/`@group_reward` contribution under `name` (weight
         applied; summed into `reward`). Warns on override — a per-rollout reward and

{verifiers-0.1.15.dev396 → verifiers-0.1.15.dev397}/verifiers/v1/types.py

@@ -160,27 +160,47 @@ class Usage(StrictBaseModel):
     reasoning_tokens: int | None = None
     cost: float | None = None
 
+    @classmethod
+    def from_openai(cls, usage: Any | None) -> "Usage | None":
+        """Build from an OpenAI chat-completion `usage` object (the `.usage` on a
+        `ChatCompletion`), or `None` when the provider reported none. Splits cache-read tokens
+        out of `prompt_tokens` and carries the reasoning subset + provider-reported `cost`."""
+        if usage is None:
+            return None
+        prompt_details = getattr(usage, "prompt_tokens_details", None)
+        cached = prompt_details.cached_tokens if prompt_details else None
+        completion_details = getattr(usage, "completion_tokens_details", None)
+        reasoning = completion_details.reasoning_tokens if completion_details else None
+        return cls(
+            prompt_tokens=usage.prompt_tokens - (cached or 0),
+            completion_tokens=usage.completion_tokens,
+            cached_input_tokens=cached,
+            reasoning_tokens=reasoning,
+            cost=getattr(usage, "cost", None),
+        )
+
     @classmethod
     def aggregate(cls, usages: Iterable["Usage"]) -> "Usage | None":
         """Sum per-response usage while preserving whether cache usage was reported."""
         values = list(usages)
         if not values:
             return None
+        # For the optional fields (cached / reasoning / cost), sum the responses that report them
+        # and yield None only when *no* response does — so one response omitting a field (e.g. a
+        # judge whose provider doesn't report reasoning or cost) doesn't null out the whole total.
         cached = [
-            usage.cached_input_tokens
-            for usage in values
-            if usage.cached_input_tokens is not None
+            u.cached_input_tokens for u in values if u.cached_input_tokens is not None
+        ]
+        reasoning = [
+            u.reasoning_tokens for u in values if u.reasoning_tokens is not None
         ]
-        reasoning = [usage.reasoning_tokens for usage in values]
-        costs = [usage.cost for usage in values]
+        costs = [u.cost for u in values if u.cost is not None]
         return cls(
             prompt_tokens=sum(usage.prompt_tokens for usage in values),
             completion_tokens=sum(usage.completion_tokens for usage in values),
             cached_input_tokens=sum(cached) if cached else None,
-            reasoning_tokens=sum(reasoning)
-            if all(v is not None for v in reasoning)
-            else None,
-            cost=sum(costs) if all(v is not None for v in costs) else None,
+            reasoning_tokens=sum(reasoning) if reasoning else None,
+            cost=sum(costs) if costs else None,
         )
 
     @property