verifiers 0.2.2.dev79__py3-none-any.whl → 0.2.2.dev81__py3-none-any.whl

verifiers/v1/cli/eval/runner.py

@@ -10,7 +10,6 @@ from verifiers.v1.cli.dashboard import dashboard
 from verifiers.v1.cli.eval import resume
 from verifiers.v1.cli.output import (
     append_episode,
-    append_trace,
     output_path,
     save_config,
 )
@@ -77,8 +76,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     write_lock = asyncio.Lock()
 
     async def on_complete(episode: Episode) -> None:
-        for trace in episode.traces:
-            trace.record_run(EvalRunInfo(id=config.uuid))
+        episode.record_run(EvalRunInfo(id=config.uuid))
         await append_episode(out, episode, write_lock)
 
     # Serving resources (shared tool servers, interception) come up once for the
@@ -258,9 +256,10 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                 )
             records = []
             for trace in traces:
-                trace.record_run(EvalRunInfo(id=config.uuid))
-                await append_trace(out, trace, write_lock, env=config.env_id)
-                records.append(Episode.of(trace))
+                record = Episode.of(trace, env=config.env_id)
+                record.record_run(EvalRunInfo(id=config.uuid))
+                await append_episode(out, record, write_lock)
+                records.append(record)
             return records
 
         async def run_unit(payload: dict) -> list[Episode]:
@@ -271,8 +270,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                     sampling=config.sampling,
                     **payload,
                 )
-            for trace in episode.traces:
-                trace.record_run(EvalRunInfo(id=config.uuid))
+            episode.record_run(EvalRunInfo(id=config.uuid))
             await append_episode(out, episode, write_lock)
             return [episode]
 

verifiers/v1/cli/output.py

@@ -12,6 +12,7 @@ before the episode atom (one bare trace per line) are still readable:
 
 import asyncio
 import json
+from functools import cache
 from pathlib import Path
 
 import tomli_w
@@ -29,6 +30,9 @@ TRACES_FILE = "traces.jsonl"
 CONFIG_FILE = "config.toml"
 """Filename a run's resolved config is written to (re-runnable via `@ config.toml`)."""
 
+# Compiling an adapter is the expensive part; run output reuses only a few model classes.
+type_adapter = cache(TypeAdapter)
+
 
 def output_path(config: EvalConfig) -> Path:
     """Where this run writes: `outputs/<env>--<model>--<harness>/<uuid>` (or the explicit
@@ -72,7 +76,7 @@ def save_config(config: BaseModel, results_dir: Path) -> None:
 def write_episode(results_dir: Path, episode: Episode) -> None:
     """Serialize and append one rollout episode in the worker thread."""
     # Preserve fields declared by typed Trace subclasses nested in the episode.
-    data = TypeAdapter(type(episode)).dump_json(episode, exclude_none=True)
+    data = type_adapter(type(episode)).dump_json(episode, exclude_none=True)
     with (results_dir / TRACES_FILE).open("ab") as f:
         f.write(data + b"\n")
 
@@ -88,7 +92,7 @@ def read_episodes(results_dir: Path, trace_type: type) -> list[Episode]:
     `trace_type` (`Trace[WireTaskData, ...]` reads any taskset's file without
     importing it). A pre-episode line (one bare trace) is wrapped as a single-trace
     record, so both file generations read uniformly."""
-    trace_adapter = TypeAdapter(trace_type)
+    trace_adapter = type_adapter(trace_type)
     episodes: list[Episode] = []
     with (results_dir / TRACES_FILE).open(encoding="utf-8") as f:
         for line in f:

verifiers/v1/configs/judge.py

@@ -5,7 +5,7 @@ from collections.abc import Sequence
 from pathlib import Path
 from typing import Any
 
-from pydantic import BaseModel, SerializeAsAny
+from pydantic import BaseModel, FiniteFloat, SerializeAsAny
 
 from verifiers.v1.clients import BaseClientConfig
 from verifiers.v1.types import ID, SamplingConfig
@@ -17,7 +17,7 @@ class JudgeConfig(BaseClientConfig):
     """Plugin id; empty for a judge called directly by task code."""
     name: str = ""
     """Reward key override for a plugged judge."""
-    weight: float = 1.0
+    weight: FiniteFloat = 1.0
     model: str = "openai/gpt-5.4-nano"
     sampling: SamplingConfig = SamplingConfig()
     prompt: Path | None = None

verifiers/v1/envs/agentic_judge/env.py

@@ -14,14 +14,18 @@ task's collected artifacts.
 """
 
 import json
-import math
 import re
-import tomllib
 from pathlib import Path
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import FiniteFloat
 
 import verifiers.v1 as vf
+from verifiers.v1.judges.rubric import (
+    Criterion,
+    RubricVerdicts,
+    load_criteria,
+    score_verdicts,
+)
 from verifiers.v1.utils.compile import validate_pairing
 
 VERDICT_FILE = "/tmp/verdict.json"
@@ -38,29 +42,6 @@ TASK_SECTION = """\
 {prompt}"""
 
 
-class Criterion(BaseModel):
-    """One rubric criterion — the plugged rubric judge's format, mirrored so the
-    same `criteria` files grade both judges."""
-
-    name: str
-    """Key for the criterion's metric (`judge/<name>`)."""
-    text: str
-    weight: float = 1.0
-    """The criterion's share of the reward."""
-    choices: list[str] = Field(default_factory=lambda: ["no", "yes"])
-    """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest
-    evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
-
-    @field_validator("choices")
-    @classmethod
-    def _check_choices(cls, v: list[str]) -> list[str]:
-        if len(v) < 2:
-            raise ValueError(f"`choices` needs at least two options, got {v}")
-        if len(set(v)) != len(v):
-            raise ValueError(f"`choices` has duplicate options: {v}")
-        return v
-
-
 SOLVED = Criterion(
     name="solved",
     text="The task is fully solved: what the task asked for is achieved, and "
@@ -210,7 +191,7 @@ class JudgeTask(vf.Task):
                 f"the judge wrote no verdict to {VERDICT_FILE}; its final act must "
                 'be writing {"verdicts": [{"name", "reason", "verdict"}, ...]} there'
             ) from e
-        trace.info["verdict"] = json.loads(raw)
+        trace.info["verdict"] = RubricVerdicts.model_validate_json(raw).model_dump()
 
 
 class TextFile(vf.BaseConfig):
@@ -259,38 +240,16 @@ class JudgeTaskConfig(vf.BaseConfig):
     def criteria(self) -> list[Criterion]:
         if self.rubric is None:
             return [SOLVED]
-        text = self.rubric.read_text(encoding="utf-8")
-        data = (
-            tomllib.loads(text)
-            if self.rubric.suffix.lower() == ".toml"
-            else json.loads(text)
-        )
-        items = data.get("criteria", []) if isinstance(data, dict) else data
-        criteria = [Criterion.model_validate(item) for item in items]
-        if not criteria:
-            raise ValueError(f"rubric file '{self.rubric}' lists no criteria")
-        names = [criterion.name for criterion in criteria]
-        if len(set(names)) != len(names):
-            raise ValueError(
-                f"rubric file '{self.rubric}' has duplicate criterion names"
-            )
-        if bad := [c.name for c in criteria if not 0 <= c.weight < math.inf]:
-            raise ValueError(
-                f"rubric '{self.rubric}' has negative or non-finite criterion "
-                f"weights: {bad}"
-            )
-        if sum(criterion.weight for criterion in criteria) <= 0:
-            raise ValueError(f"rubric '{self.rubric}' has no positive criterion weight")
-        return criteria
+        return load_criteria(self.rubric)
 
 
 class ScoreConfig(vf.BaseConfig):
     """How the judge's verdict composes with the taskset's own rewards on the
     solver's trace. Judge-only by default."""
 
-    task_weight: float = 0.0
+    task_weight: FiniteFloat = 0.0
     """Scale applied to the taskset's own rewards; 1 keeps them next to the verdict."""
-    judge_weight: float = 1.0
+    judge_weight: FiniteFloat = 1.0
     """Weight of the judge's verdict in the solver's reward."""
 
 
@@ -363,37 +322,9 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         if "judge" not in by_agent:
             return
         solution, verdict = by_agent["solver"], by_agent["judge"]
-        data = verdict.info.get("verdict")
-        if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list):
-            raise TypeError(
-                f"no verdicts on the judge's trace (expected {VERDICT_FILE} with a "
-                '"verdicts" list)'
-            )
+        verdicts = RubricVerdicts.model_validate(verdict.info.get("verdict")).verdicts
         criteria = self.config.task.criteria()
-        by_criterion = {c.name: c for c in criteria}
-        answers: dict[str, str] = {}
-        for entry in data["verdicts"]:
-            if not isinstance(entry, dict):
-                raise TypeError(f"verdict entry {entry!r} is not an object")
-            name = str(entry.get("name"))
-            if name in answers:
-                # Contradictory duplicates must not collapse to whichever came last.
-                raise ValueError(f"judge answered criterion {name!r} more than once")
-            answers[name] = str(entry.get("verdict"))
-        if sorted(answers) != sorted(by_criterion):
-            raise ValueError(
-                f"judge verdicts name {sorted(answers)}, expected the rubric's "
-                f"{sorted(by_criterion)}"
-            )
-        scores: dict[str, float] = {}
-        for name, answer in answers.items():
-            choices = by_criterion[name].choices
-            # An off-menu answer is a judge failure, not a zero score.
-            if answer not in choices:
-                raise ValueError(
-                    f"judge answered {answer!r} for '{name}', expected one of {choices}"
-                )
-            scores[name] = choices.index(answer) / (len(choices) - 1)
+        scores = score_verdicts(verdicts, criteria, "the rubric's")
         for criterion in criteria:
             solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
         if self.config.score.task_weight != 1.0:

verifiers/v1/episode.py

@@ -1,14 +1,14 @@
 """The episode — one run's traces plus their shared standing, whole."""
 
 import uuid
-from typing import Generic
+from typing import Any, Generic
 
 from pydantic import BaseModel, Field
 
 from verifiers.v1.configs.agent import WireAgentConfig
 from verifiers.v1.state import State, StateT
 from verifiers.v1.task import DataT, WireTaskData
-from verifiers.v1.trace import AgentConfigT, Error, Trace
+from verifiers.v1.trace import AgentConfigT, Error, RunInfo, Trace
 from verifiers.v1.types import Usage
 
 
@@ -26,12 +26,19 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     env: EnvInfo = Field(default_factory=EnvInfo)
     """The env that produced this episode."""
+    run: RunInfo | None = None
+    """The run this episode belongs to (eval or train), consumer-stamped. It lives here rather than
+    on each trace because the episode is what a consumer dispatches, and an episode that produced
+    no traces would otherwise have nowhere to say which run it was."""
     ok: bool = False
     """Whether the episode completed successfully."""
     errors: list[Error] = Field(default_factory=list)
     """Every error captured across attempts, oldest to newest."""
     traces: list[Trace[DataT, StateT, AgentConfigT]] = Field(default_factory=list)
     """Every agent's trace, in completion order."""
+    info: dict[str, Any] = Field(default_factory=dict)
+    """Scratch space for episode-level metadata, the counterpart to `Trace.info`. What describes
+    the whole episode belongs here rather than repeated on each of its traces."""
 
     @property
     def last_error(self) -> Error | None:
@@ -72,6 +79,13 @@ class Episode(BaseModel, Generic[DataT, StateT, AgentConfigT]):
             grouped.setdefault(trace.agent.name, []).append(trace)
         return grouped
 
+    def record_run(self, run: RunInfo | None = None, **info: Any) -> None:
+        """Record the run identity and any extra metadata about this episode. Both describe the
+        episode as a whole, so they are recorded once here rather than repeated on every trace."""
+        if run is not None:
+            self.run = run
+        self.info.update(info)
+
     @classmethod
     def of(cls, trace: Trace, env: str = "") -> "Episode":
         """The single-agent record: one trace as its own episode."""

verifiers/v1/harnesses/rlm/harness.py

@@ -6,7 +6,7 @@ import random
 import shlex
 from typing import Literal
 
-from pydantic import Field, model_validator
+from pydantic import Field, PositiveInt, model_validator
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.harness import HarnessConfig
@@ -38,26 +38,18 @@ class RLMHarnessConfig(HarnessConfig):
     builtin_skills: list[BuiltinSkill] = Field(default_factory=list)
     """Built-in rlm skills to enable (RLM_SKILLS), e.g. `["edit"]`; empty enables none.
     The tool set is fixed (ipython); the base `skills` field takes SKILL.md paths."""
-    summarize_at_tokens: int | tuple[int, int] | None = None
+    summarize_at_tokens: PositiveInt | tuple[PositiveInt, PositiveInt] | None = None
     """Auto-compaction threshold (RLM_SUMMARIZE_AT_TOKENS): compact the context once it grows
     past this many tokens. An int is a fixed threshold; a `(lo, hi)` pair draws a per-group
     threshold (seeded by the task index, so a task's rollouts share one draw and tasks vary).
     `None` disables auto-compaction; ints must be positive."""
 
     @model_validator(mode="after")
-    def validate_limits(self) -> "RLMHarnessConfig":
+    def validate_range(self) -> "RLMHarnessConfig":
         value = self.summarize_at_tokens
-        if isinstance(value, tuple):
-            lo, hi = value
-            if lo <= 0 or hi <= 0:
-                raise ValueError("`summarize_at_tokens` range bounds must be positive.")
-            if lo > hi:
-                raise ValueError(
-                    "`summarize_at_tokens` range must be (lo, hi) with lo <= hi."
-                )
-        elif value is not None and value <= 0:
+        if isinstance(value, tuple) and value[0] > value[1]:
             raise ValueError(
-                "`summarize_at_tokens` must be positive, or None to disable."
+                "`summarize_at_tokens` range must be (lo, hi) with lo <= hi."
             )
         return self
 

verifiers/v1/interception/server.py

@@ -30,7 +30,7 @@ from contextlib import asynccontextmanager
 from typing import Literal
 
 from aiohttp import web
-from pydantic import TypeAdapter, ValidationError
+from pydantic import ValidationError
 from pydantic_core import PydanticSerializationError, from_json, to_json
 
 from verifiers.v1 import graph
@@ -737,7 +737,7 @@ class InterceptionServer(Interception):
         state = session.trace.state
         return web.Response(
             # TypeAdapter emits UTF-8 bytes directly, avoiding a JSON str copy in aiohttp.
-            body=TypeAdapter(type(state)).dump_json(state),
+            body=session.state_adapter.dump_json(state),
             content_type="application/json",
             charset="utf-8",
         )
@@ -768,7 +768,7 @@ class InterceptionServer(Interception):
         state_cls = type(session.trace.state)
         raw = await request.read()
         try:
-            new_state = state_cls.model_validate_json(raw)
+            new_state = session.state_adapter.validate_json(raw)
         except ValidationError as e:
             # Reject malformed, over-nested, or mismatched state before it enters the shared channel.
             logger.warning("state PUT rejected: id=%s %s", session.trace.id, e)

verifiers/v1/judges/rubric.py

@@ -2,14 +2,13 @@
 
 import asyncio
 import json
-import math
 import re
 import tomllib
 from functools import cached_property
 from pathlib import Path
-from typing import cast
+from typing import Annotated, cast
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, TypeAdapter, field_validator
 
 from verifiers.v1.configs.judge import JudgeConfig
 from verifiers.v1.judge import Judge, JudgeView, judge_question, judge_response
@@ -17,6 +16,8 @@ from verifiers.v1.task import TaskData
 from verifiers.v1.trace import Trace
 from verifiers.v1.types import ID
 
+CriterionWeight = Annotated[float, Field(ge=0, allow_inf_nan=False)]
+
 RUBRIC_PROMPT = (Path(__file__).resolve().parent / "rubric.txt").read_text(
     encoding="utf-8"
 )
@@ -60,29 +61,62 @@ class Criterion(BaseModel):
     name: str
     """Key for the criterion's metric (`<judge name>/<name>`) and its `weights` override."""
     text: str
-    weight: float = 1.0
+    weight: CriterionWeight = 1.0
     """The criterion's share of the reward (overridable per name via `weights` in config)."""
-    choices: list[str] = Field(default_factory=lambda: ["no", "yes"])
+    choices: list[str] = Field(default_factory=lambda: ["no", "yes"], min_length=2)
     """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest
     evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
 
     @field_validator("choices")
     @classmethod
     def _check_choices(cls, v: list[str]) -> list[str]:
-        if len(v) < 2:
-            raise ValueError(f"`choices` needs at least two options, got {v}")
         if len(set(v)) != len(v):
             raise ValueError(f"`choices` has duplicate options: {v}")
         return v
 
 
+CRITERIA_ADAPTER = TypeAdapter(list[Criterion])
+
+
+def load_criteria(
+    path: Path, weights: dict[str, CriterionWeight] | None = None
+) -> list[Criterion]:
+    """Load a JSON or TOML rubric and apply validated config overrides."""
+    text = path.read_text(encoding="utf-8")
+    data = tomllib.loads(text) if path.suffix.lower() == ".toml" else json.loads(text)
+    items = data.get("criteria", []) if isinstance(data, dict) else data
+    criteria = CRITERIA_ADAPTER.validate_python(items)
+    if not criteria:
+        raise ValueError(f"rubric file '{path}' lists no criteria")
+    names = [criterion.name for criterion in criteria]
+    if len(set(names)) != len(names):
+        raise ValueError(f"rubric file '{path}' has duplicate criterion names")
+    overrides = weights or {}
+    if unknown := set(overrides) - set(names):
+        raise ValueError(
+            f"`weights` overrides name no criterion in '{path}': {sorted(unknown)}"
+        )
+    criteria = [
+        criterion.model_copy(
+            update={"weight": overrides.get(criterion.name, criterion.weight)}
+        )
+        for criterion in criteria
+    ]
+    total = sum(criterion.weight for criterion in criteria)
+    if total == float("inf"):
+        raise ValueError(f"rubric '{path}' has a non-finite total criterion weight")
+    if total <= 0:
+        raise ValueError(f"rubric '{path}' has no positive criterion weight")
+    return criteria
+
+
 class RubricJudgeConfig(JudgeConfig):
     id: ID = "rubric"
     """Pinned to the built-in, so a code-level default entry needs no explicit id."""
     path: Path
     """A `.toml` or `.json` file containing a `criteria` list. Relative paths resolve
     against the evaluation's working directory."""
-    weights: dict[str, float] = Field(default_factory=dict)
+    weights: dict[str, CriterionWeight] = Field(default_factory=dict)
     """Per-criterion weight overrides by criterion name (config wins over the file)."""
     question_field: str = ""
     """Task field to fill the prompt's `{question}`; empty = the task's prompt rendered as
@@ -96,7 +130,7 @@ class RubricJudgeConfig(JudgeConfig):
     """How much of the rollout fills `{response}` (see `JudgeView`). Defaults to the whole
     transcript — rubric criteria typically grade the process (tool use, citations,
     intermediate steps), not just the final answer."""
-    max_criteria: int | None = None
+    max_criteria: int | None = Field(default=None, ge=1)
     """How many criteria to grade per judge call. `None` (default) grades all criteria in one
     call. `1` sends one call per criterion (n independent judges); `k` batches them k-at-a-time.
     Batches are graded concurrently and merged. Smaller batches trade more calls for focus/
@@ -119,45 +153,43 @@ class RubricVerdicts(BaseModel):
     verdicts: list[CriterionVerdict]
 
 
+def score_verdicts(
+    verdicts: list[CriterionVerdict],
+    criteria: list[Criterion],
+    expected: str,
+) -> dict[str, float]:
+    """Validate a complete named verdict set and normalize its ordered choices."""
+    by_criterion = {criterion.name: criterion for criterion in criteria}
+    answers: dict[str, str] = {}
+    for verdict in verdicts:
+        if verdict.name in answers:
+            raise ValueError(
+                f"judge answered criterion {verdict.name!r} more than once"
+            )
+        answers[verdict.name] = verdict.verdict
+    if sorted(answers) != sorted(by_criterion):
+        raise ValueError(
+            f"judge verdicts name {sorted(answers)}, expected {expected} "
+            f"{sorted(by_criterion)}"
+        )
+    scores: dict[str, float] = {}
+    for name, answer in answers.items():
+        choices = by_criterion[name].choices
+        if answer not in choices:
+            raise ValueError(
+                f"judge answered {answer!r} for '{name}', expected one of {choices}"
+            )
+        scores[name] = normalize_choice(answer, choices)
+    return scores
+
+
 class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
     prompt = RUBRIC_PROMPT
     schema = RubricVerdicts
 
     @cached_property
     def criteria(self) -> list[Criterion]:
-        path = self.config.path
-        text = path.read_text(encoding="utf-8")
-        data = (
-            tomllib.loads(text) if path.suffix.lower() == ".toml" else json.loads(text)
-        )
-        items = data.get("criteria", []) if isinstance(data, dict) else data
-        criteria = [Criterion.model_validate(item) for item in items]
-        if not criteria:
-            raise ValueError(f"rubric file '{path}' lists no criteria")
-        names = [criterion.name for criterion in criteria]
-        if len(set(names)) != len(names):
-            raise ValueError(f"rubric file '{path}' has duplicate criterion names")
-        if unknown := set(self.config.weights) - set(names):
-            raise ValueError(
-                f"`weights` overrides name no criterion in '{path}': {sorted(unknown)}"
-            )
-        criteria = [
-            criterion.model_copy(
-                update={
-                    "weight": self.config.weights.get(criterion.name, criterion.weight)
-                }
-            )
-            for criterion in criteria
-        ]
-        if bad := [c.name for c in criteria if not 0 <= c.weight < math.inf]:
-            # A negative weight would invert a criterion (pushing the reward out of [0, 1]);
-            # NaN/inf (which json.loads accepts) would corrupt the weighted mean.
-            raise ValueError(
-                f"rubric '{path}' has negative or non-finite criterion weights: {bad}"
-            )
-        if sum(criterion.weight for criterion in criteria) <= 0:
-            raise ValueError(f"rubric '{path}' has no positive criterion weight")
-        return criteria
+        return load_criteria(self.config.path, self.config.weights)
 
     async def grade_batch(
         self, task: TaskData, trace: Trace, batch: list[Criterion]
@@ -205,30 +237,12 @@ class RubricJudge(Judge[RubricVerdicts, RubricJudgeConfig]):
                     f"judge returned no verdicts JSON object: {result.text!r}"
                 )
             verdicts = RubricVerdicts.model_validate(obj).verdicts
-        # Exactly one verdict per criterion in the batch, matched by name — anything else is a
-        # judge failure and must error the rollout, not score the model (see `judge_verdict`).
-        by_criterion = {c.name: c for c in batch}
-        if sorted(v.name for v in verdicts) != sorted(by_criterion):
-            raise ValueError(
-                f"judge verdicts name {sorted(v.name for v in verdicts)}, expected the "
-                f"batch's {sorted(by_criterion)}"
-            )
-        scores: dict[str, float] = {}
-        for v in verdicts:
-            choices = by_criterion[v.name].choices
-            # An off-menu answer is a judge failure, not a zero score.
-            if v.verdict not in choices:
-                raise ValueError(
-                    f"judge answered {v.verdict!r} for '{v.name}', expected one of {choices}"
-                )
-            scores[v.name] = normalize_choice(v.verdict, choices)
-        return scores
+        # A malformed verdict is a judge failure and must error the rollout, not score the model.
+        return score_verdicts(verdicts, batch, "the batch's")
 
     async def score(self, task: TaskData, trace: Trace) -> float:
         criteria = self.criteria
         k = self.config.max_criteria
-        if k is not None and k < 1:
-            raise ValueError(f"`max_criteria` must be >= 1 or None, got {k}")
         batches = (
             [criteria]
             if k is None

verifiers/v1/legacy.py

@@ -16,11 +16,11 @@ v1 stays importable without the v0 package present.
 import contextlib
 import logging
 from pathlib import Path
-from typing import Any
+from typing import Annotated, Any
 
 import zmq
 import zmq.asyncio
-from pydantic import ValidationError
+from pydantic import BeforeValidator, OnErrorOmit, TypeAdapter, ValidationError
 
 from verifiers.v1 import graph
 from verifiers.v1.configs.agent import AgentConfig
@@ -76,19 +76,19 @@ def _as_dict(obj: Any) -> Any:
     return obj
 
 
+TOOLS_ADAPTER = TypeAdapter(
+    list[OnErrorOmit[Annotated[Tool, BeforeValidator(_as_dict)]]]
+)
+
+
 def _to_v1_tools(raw: Any) -> list[Tool] | None:
     """Map v0 ``RolloutOutput.tool_defs`` onto ``Trace.tools``. The v0 and v1 ``Tool``
     shapes are identical (name/description/parameters/strict), so this is a re-validation;
     malformed entries are dropped rather than failing the whole trace mapping."""
-    defs: list[Tool] = []
-    for t in raw or []:
-        t = _as_dict(t)
-        if not isinstance(t, dict):
-            continue
-        try:
-            defs.append(Tool.model_validate(t))
-        except ValidationError:
-            continue
+    try:
+        defs = TOOLS_ADAPTER.validate_python(raw or [])
+    except ValidationError:
+        return None
     return defs or None
 
 
@@ -276,8 +276,8 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
         # base task type above.
         agent=AgentInfo(config=AgentConfig()),
         tools=_to_v1_tools(out.get("tool_defs")) or [],
-        rewards={"reward": Reward(score=float(out.get("reward") or 0.0))},
-        metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
+        rewards={"reward": Reward(score=out.get("reward") or 0.0)},
+        metrics=out.get("metrics") or {},
         info=dict(out.get("info") or {}),
         is_completed=bool(out.get("is_completed", True)),
         # Bridged rollouts are complete by construction; the sentinel mirrors

verifiers/v1/session.py

@@ -11,8 +11,11 @@ import asyncio
 import logging
 from collections.abc import Awaitable, Callable
 from dataclasses import dataclass, field
+from functools import cached_property
 from typing import TYPE_CHECKING
 
+from pydantic import TypeAdapter
+
 from verifiers.v1.clients import Client, ModelContext
 from verifiers.v1.trace import Trace
 
@@ -96,6 +99,11 @@ class RolloutSession:
     its client disconnects, so a request whose program died at teardown would keep driving
     the exchange (upstream call, simulator turn) — unregistering cancels these instead."""
 
+    @cached_property
+    def state_adapter(self) -> TypeAdapter:
+        """The rollout's state codec, built only when a state channel is used."""
+        return TypeAdapter(type(self.trace.state))
+
     def adopt(self, task: "asyncio.Task | None") -> None:
         """Track a handler task serving this session, for cancellation at release.
         Callers adopt in the same synchronous stretch that fetched the session, so

verifiers/v1/trace.py

@@ -304,9 +304,6 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     """Unique ID for this trace, auto-generated."""
     verifiers: VersionInfo = Field(default_factory=_current_build)
     """The verifiers version that produced this trace."""
-    run: RunInfo | None = None
-    """The run this trace belongs to (eval or train), consumer-stamped."""
-
     task: TraceTask[DataT]
     """The task data that seeded this trace."""
     agent: AgentInfo[AgentConfigT]
@@ -480,12 +477,6 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
         if response.usage is not None:
             self.extra_usage.append(response.usage)
 
-    def record_run(self, run: RunInfo | None = None, **info: Any) -> None:
-        """Record the run identity (eval / train), and optional extra info."""
-        if run is not None:
-            self.run = run
-        self.info.update(info)
-
     def stop(self, condition: str) -> None:
         """Stop the trace, optionally with a stop condition."""
         self.is_completed = True

{verifiers-0.2.2.dev79.dist-info → verifiers-0.2.2.dev81.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev79
+Version: 0.2.2.dev81
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev79.dist-info → verifiers-0.2.2.dev81.dist-info}/RECORD

@@ -170,18 +170,18 @@ verifiers/utils/version_utils.py,sha256=39gV8-mNF9GspAgKloLo8oy7_MVluMrwGPrntQv2
 verifiers/v1/__init__.py,sha256=v6qrJQFeNL2b4wd0PsDWiituw2Y-f5rMaFmdbdP6SdM,7505
 verifiers/v1/agent.py,sha256=MQVKGjUhrP8uzRXAq94uJHkEsdHFziPxQHZ7TOD324I,31306
 verifiers/v1/env.py,sha256=g6fpoG-Z9lM3vTKrrU-QRJY3S88_y9zsPHwBvfs5vdY,17865
-verifiers/v1/episode.py,sha256=YtuRTV_PpwDglXFeN6UNg8IswpP1HseFLCYtascGMNI,3218
+verifiers/v1/episode.py,sha256=8G5OaGOku9S_gavQdpfUZgRZfrf1zmpjFrBL-TsppIk,4123
 verifiers/v1/errors.py,sha256=kQeEPX06TwAIuiz7MlWFYirrpKbx5STi6A5b_dtQQoA,6885
 verifiers/v1/graph.py,sha256=aBAzh1Ibp3klKNYjvzNyJSbjV8Oi5TJ791dN7o6ex1E,29070
 verifiers/v1/harness.py,sha256=eKuqvYbQjnCMavkiYWZQDHJgH08m6SJ4PuevPuShjS0,11097
 verifiers/v1/judge.py,sha256=Pvr0C41ah1qkNSZ0WXdkDvBMfv5YSfY1REHncwlJ8cg,9346
-verifiers/v1/legacy.py,sha256=eLsFiwihOMh4zS_u1V7DXF22r9beoG4k2JGwNEcSPfg,22872
+verifiers/v1/legacy.py,sha256=YvcWF6d8xvkeqUPM36oif0ZrjmB0mj7nNrfwha0Nr2w,22850
 verifiers/v1/rollout.py,sha256=s1f1VZAAAI89j3y9ceq_E4mvX0F7cFQAnyx27SZsjvA,17881
-verifiers/v1/session.py,sha256=tNAraQfiJKpWqgBFNLyPhfHiw0JhzzkyOfRcKWGnR2Q,6939
+verifiers/v1/session.py,sha256=p8vz89DJUmb9r8dkq80zn2WBEqo8Vwh8ot1ZXm_WVAM,7210
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
 verifiers/v1/task.py,sha256=jwMKiKlMtksTd8j2dcDlFtb6LC8BRE-N5jkNXlMp2jc,9457
 verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
-verifiers/v1/trace.py,sha256=Z4uK-lYAbl-6CHFhTo-gadHnbi63LC0sJtUERC_m5ss,19477
+verifiers/v1/trace.py,sha256=esJE1aT9jRCbBad-BVR9xjX5Emk4mHdYmC7Zct-3zK8,19129
 verifiers/v1/types.py,sha256=1PxamJspmoTc3OlFZAwH6o_2i_Tg9ZNvHxttIHWIxq8,8948
 verifiers/v1/acp/__init__.py,sha256=9SYCFtzGUM_wH98ldpVLcFhoq2M999jbG0tU5ODY70U,2297
 verifiers/v1/acp/_runner.py,sha256=BcYaNZHuawzCgxOVdhiF6PY_B1HMxIA1MwHy0mOzhSk,7740
@@ -189,7 +189,7 @@ verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 verifiers/v1/cli/debug.py,sha256=WGCiapH2CCwyPfESca5623-_csQv93PoglR8yX5-D3g,11488
 verifiers/v1/cli/gepa.py,sha256=hLIbS_4jBx0vvy4PXoePR5riFXcNvzQCMixZ-g2OR_U,4026
 verifiers/v1/cli/init.py,sha256=Qj96Kwzfhaejhrc0yYqHNRl3ncoID028RGlHvS1C98Q,8508
-verifiers/v1/cli/output.py,sha256=pMS5Z0EBgyAVShNCAk9jVURNeLOaTfeRWpt0J8hHjq8,5890
+verifiers/v1/cli/output.py,sha256=JiNjMO2kwM4p3zMPg3k4spuJbbE7-jOAtYBttPap7qI,6045
 verifiers/v1/cli/replay.py,sha256=DAh9kB5PB-ppZLGhxdJwrCPch_Yky5HKhplwGFhKEz0,10164
 verifiers/v1/cli/resolve.py,sha256=OsNw4C3r9xOVFa-mminJFcex_bAsJ6EQ8K7A_H8u2kQ,4370
 verifiers/v1/cli/validate.py,sha256=7ax6FqIzNBSYjd3JXK4v7Vbq34Rozyh1OvGpYnGqlSg,10266
@@ -201,7 +201,7 @@ verifiers/v1/cli/dashboard/validate.py,sha256=rLsQ_31DjTIZpC_VO7zSG41ncUEBxqCx8m
 verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/cli/eval/main.py,sha256=xiMMIIbsy77SEOryHXOQyswS9WyrPKTyBwW65W9WTjI,5511
 verifiers/v1/cli/eval/resume.py,sha256=QwXuLPs2lN-aMZk1CyaT12egRPzeI6vfY39DAd0nYLU,7171
-verifiers/v1/cli/eval/runner.py,sha256=iR_Bapy5r0IC43GpzlfKIO9RAzFmKcY6EMcwYbEJupY,12181
+verifiers/v1/cli/eval/runner.py,sha256=oAMWf6uwuS_tEyE6OY_Q0-9_TTVCduKC-RJu797QFEc,12117
 verifiers/v1/clients/__init__.py,sha256=Ysig0tE_0E4Jsfgfes1XHN-fK1s_RCXdqZD6E7lK4PU,507
 verifiers/v1/clients/base.py,sha256=hhtTumKC7lgI6cj7Hks91DCjqNTwAh40DD_j8CWGd6c,1560
 verifiers/v1/clients/client.py,sha256=L4axOl-dF9_MSYagB_xAO5qbeE53jNZJnB39VLdVBwo,3156
@@ -212,7 +212,7 @@ verifiers/v1/configs/agent.py,sha256=V_2FDvPHRB3_bHN9QnNnDYOjlmTP7NtG22SaopU-X64
 verifiers/v1/configs/client.py,sha256=dCi2aC_wVHCrM0xRi9foFcbqp9pN2GR7yxEs1z_ChJM,4371
 verifiers/v1/configs/env.py,sha256=7ESsWZRWJpKqR5EEV1rmj0R1Dkn2Pn8MVSXPJOzRiF8,7885
 verifiers/v1/configs/harness.py,sha256=uDBb3DoX3RGbH2fOAvVVMaluP3VivtMfurGiBbbpuAs,1695
-verifiers/v1/configs/judge.py,sha256=O7oe28tS24fKRXUML_-6vHrVGdG6w7061SPQXOH7XvM,2223
+verifiers/v1/configs/judge.py,sha256=jc9w8eYsqSEyMMY7n12EWyYcwqQl5s-47xV3Me7WA0c,2242
 verifiers/v1/configs/legacy.py,sha256=YOJM6L5u6-jYpZ1CLpPInufKdmz1lEsMNKIRdT2JaaY,1080
 verifiers/v1/configs/retries.py,sha256=-nPlgmz7J_NZ4GOYG6oCc3thXo4La2SufBmvQGvvPGo,821
 verifiers/v1/configs/serve.py,sha256=nSoJcTX2S0Vy9EUeNqW0Yo-f9_MUqpBiY-Q9FF_4cRo,2576
@@ -232,7 +232,7 @@ verifiers/v1/dialects/chat.py,sha256=R_otqi5N0snB1QVKIYloNeLHR9kc7OmI06HYg1ir_Lw
 verifiers/v1/dialects/responses.py,sha256=ismnxUikPcPUIm80FUTPBbj-Cn1e0yGAsISMEseAWRc,13679
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=6vwtMCQ_cuvuubCF5-nrE5W7gGgzE-LIPKorkM5DXdw,309
-verifiers/v1/envs/agentic_judge/env.py,sha256=6QrfuIlg-9MkcqIn5_6F3qJ6ddLJNaeYsZRJC9EzGkw,17011
+verifiers/v1/envs/agentic_judge/env.py,sha256=0KXuTEg4nd0jprPqFgQYFxf2JLdPVbUmljbgPOT-5Bg,13810
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZnLMje6eVgyz5U,138
@@ -274,14 +274,14 @@ verifiers/v1/harnesses/pi/harness.py,sha256=_O-pyg2RM23-RzQosKHKf8uG8KflS4OrWSsE
 verifiers/v1/harnesses/pool/__init__.py,sha256=HTYsNiWGdiNEsSHj4xhPmshJSNb7SHjQ7d4yZQ4p3HM,127
 verifiers/v1/harnesses/pool/harness.py,sha256=6DlOA2W3s-NrxdQ9GaMmZg7mSNJGz2GC21kH2Pmoy9w,4025
 verifiers/v1/harnesses/rlm/__init__.py,sha256=hwx51xTdEWzzYgMj-p4ETNcptoub3IPMDbEdJ4uB8Ns,122
-verifiers/v1/harnesses/rlm/harness.py,sha256=M7Smj6ySz6o-SomN5UORwNjzUTEs_V0g_BSiV2Jj2qQ,7214
+verifiers/v1/harnesses/rlm/harness.py,sha256=icZ91FrylW5U8S0sQ178aGJSYyf6eH2JYZvmIKy0L4c,6918
 verifiers/v1/harnesses/terminus_2/__init__.py,sha256=l1pwfg1HyVDEqT9efdnFWcXqaZwtKwe7TrplkyxZSA0,166
 verifiers/v1/harnesses/terminus_2/harness.py,sha256=h-W2Yc4juKcg9YuGp7sZtJghTdu-S6pq4Wf5cP0VBeE,3027
 verifiers/v1/harnesses/terminus_2/program.py,sha256=XWflwULPfVUyrwAldXLRVOoUWoWW9bYNsaygWef0Sfg,2668
 verifiers/v1/interception/__init__.py,sha256=5Kvo7tcBUVaa65s9A1NiOH3CPpzw-P5btXwxB4Nbbvs,4251
 verifiers/v1/interception/base.py,sha256=Mg16Y104lyZilF9shdivdqYoqz4OYp978i_myyrulu8,2529
 verifiers/v1/interception/pool.py,sha256=sk_zcdTV8EVRuzqlYMXxon_42tSDAlFvwcyOO2EF7hw,6473
-verifiers/v1/interception/server.py,sha256=BUaQXvvJDJK58OMf39oLw6VvLnXlHNRq3gh22eMucOo,36758
+verifiers/v1/interception/server.py,sha256=siy2WizKHsTyM2LiigQVcrTR2oACQsW4nclmiyF3m2M,36748
 verifiers/v1/interception/tunnel/__init__.py,sha256=eVNZJszj6myrKhx9jmJz7VRd9qlfj0AUvUZwI2bXEj8,893
 verifiers/v1/interception/tunnel/base.py,sha256=DZB6uPLwM4Qy7n7m0vKeyg-Px87MhsmpYTpe2vpPEng,1998
 verifiers/v1/interception/tunnel/custom.py,sha256=yL4UbGf4bAuMsxk42mEP-t1wITQCT8BE9FHQIz0qRMA,1708
@@ -289,7 +289,7 @@ verifiers/v1/interception/tunnel/prime.py,sha256=PP3Be3PfbfsLlhF4FfpsZt7IkiYiC1D
 verifiers/v1/judges/__init__.py,sha256=MUIBWcx6c70BykrTDHhB0ITIyJPxXe6xDBou4VM7jDQ,286
 verifiers/v1/judges/reference.py,sha256=iEVHw-iJ6dbwLOOqrvWrEMJOcUEYJ0YspT04rZz7Sd4,3868
 verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93V6A8,353
-verifiers/v1/judges/rubric.py,sha256=PglrJhMQR6scFyfz20jVlLQEzeqXKV2P6r0K0K7jjBE,11916
+verifiers/v1/judges/rubric.py,sha256=qwBjJCWfaUUcQA9A4EXFQK0NumGjoevSFT2HjgBZut4,12019
 verifiers/v1/judges/rubric.txt,sha256=3KA2ZeYYdcDweJhUGopxu9EFgARhu2aWft5S5cj8Bc8,323
 verifiers/v1/mcp/__init__.py,sha256=pm8ZWBiL1q1tY4wHvJzM_QrZ9oR8aGm-bIRWiRJsgHI,408
 verifiers/v1/mcp/launch.py,sha256=wBpyeiOO86l1H2lVMbLVSV9Lbp90XeyXqauuK8snfEU,20171
@@ -337,8 +337,8 @@ verifiers/v1/utils/platform.py,sha256=56Ixmk1SER6q5LvzyYcA0hgzGpzhxKZL8Hiqp4c_XV
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev79.dist-info/METADATA,sha256=UcbhgbyAo1IsvDP_DNeoDRCydNre7663FBU_U4WzJ0E,4545
-verifiers-0.2.2.dev79.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev79.dist-info/entry_points.txt,sha256=v6v0QT9vVExnfn4br42MostI_o3f3ZbWYlTormz-U3g,515
-verifiers-0.2.2.dev79.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev79.dist-info/RECORD,,
+verifiers-0.2.2.dev81.dist-info/METADATA,sha256=RsTZSMxzSbJsukFkrvjA45a6IZtUOBxWSpfT6cs1LnU,4545
+verifiers-0.2.2.dev81.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev81.dist-info/entry_points.txt,sha256=v6v0QT9vVExnfn4br42MostI_o3f3ZbWYlTormz-U3g,515
+verifiers-0.2.2.dev81.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev81.dist-info/RECORD,,