verifiers 0.2.2.dev25__tar.gz → 0.2.2.dev27__tar.gz

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev25
+Version: 0.2.2.dev27
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/tests/v1/test_e2e.py

@@ -197,7 +197,7 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     )
     assert trace.ok, trace.errors
     assert trace.stop_condition == "user_closed"
-    assert trace.rewards["resumed"] == 1.0
+    assert trace.rewards["resumed"].score == 1.0
     assert trace.tools  # ACP-native tools, or Pi's MCP adapter meta-tool
     segments = trace.info["acp_segments"]
     assert len(segments) == 2
@@ -320,7 +320,7 @@ async def test_rubric_judge(run_v1, tmp_path):
         max_turns=2,
     )
     assert trace.ok
-    assert trace.rewards["rubric"] > 0  # the judge's verdict landed in the reward
+    assert trace.rewards["rubric"].score > 0  # the judge's verdict landed in the reward
     assert trace.metrics["rubric/always_yes"] == 1.0
     assert trace.info["judge"]  # the call was recorded onto the trace
 
@@ -392,23 +392,32 @@ async def test_env_id_best_of_n(run_v1, tmp_path):
 
 @pytest.mark.e2e
 async def test_env_id_agentic_judge(run_v1, tmp_path):
-    """The agentic judge over the echo taskset (needs docker): the judge lands in
-    its own box with the graded transcript uploaded, investigates with real
-    execution, and its parsed verdict lands on the solver's trace under the spec's
-    reward key. Wiring, not taste: the judge followed the verdict-file contract's
-    output contract — the grade itself is the model's call."""
+    """The agentic judge over the echo taskset (needs docker): the box is
+    provisioned once from the solver's runtime policy, the solver plays in it,
+    the judge lands in the SAME box with the graded trace uploaded,
+    investigates with real execution, and its parsed verdict
+    lands on the solver's trace under the spec's reward key. Wiring, not taste:
+    the judge followed the verdict-file contract — the grade itself is the
+    model's call. Exercises the config surface too: a policy-only prompt
+    override (the verdict contract is appended regardless) and
+    reward-composition weights."""
     traces = await run_v1(
         "echo-v1",
         harness=None,  # seats pin their own harness; there is no run-level one
         env={
             "id": "agentic-judge",
-            "solver": {"harness": {"id": "null"}},
-            # The judge reads the transcript and reasons before it writes the
+            # The solver owns the shared box, so the container is pinned here.
+            "solver": {"harness": {"id": "bash"}, "runtime": {"type": "docker"}},
+            # The judge reads the trace and reasons before it writes the
             # verdict file; the shared 2048-token run cap truncates it mid-audit.
             "judge": {
-                "runtime": {"type": "docker"},
+                "harness": {"id": "bash"},
                 "max_output_tokens": 8192,
             },
+            "task": {
+                "prompt": "Check EMPIRICALLY that the agent echoed the word back.",
+            },
+            "score": {"task_weight": 0.5},
         },
         output_dir=tmp_path,
         max_turns=10,
@@ -419,9 +428,11 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     (judge,) = [t for t in traces if t.agent_name == "judge"]
     assert solver.ok and judge.ok
     assert judge.trainable is False
-    assert solver.rewards["echoed"] == 1.0  # the task's own reward still runs
+    # The task's own reward keeps its raw score; the rescale lands on the weight.
+    assert solver.rewards["echoed"].score == 1.0
+    assert solver.rewards["echoed"].weight == 0.5
     assert isinstance(judge.info.get("verdict"), dict)  # scraped off the box
-    assert 0.0 <= solver.rewards["judge"] <= 1.0
+    assert 0.0 <= solver.rewards["judge"].score <= 1.0
 
 
 @pytest.mark.e2e
@@ -482,7 +493,7 @@ async def test_env_id_user_sim_with_tools(run_v1, tmp_path):
     assert assistant.ok and user.ok
     assert assistant.task.data.prompt is None  # the scenario stayed off the wire
     assert user.num_turns >= 1  # the modeled user actually drove the exchange
-    assert assistant.rewards["echoed"] == 1.0  # the tool ran, mid-conversation
+    assert assistant.rewards["echoed"].score == 1.0  # the tool ran, mid-conversation
     # The tool was advertised to the masked chat exactly as to any run.
     assert assistant.tools is not None
     assert any(tool.name == "echo_back" for tool in assistant.tools)
@@ -504,7 +515,7 @@ async def test_kuhn_poker_self_play(run_v1, tmp_path):
         rollout_timeout=300,
     )
     assert sorted(t.agent_name for t in traces) == ["player0", "player1"]
-    payoffs = {t.agent_name: t.rewards["payoff"] for t in traces}
+    payoffs = {t.agent_name: t.rewards["payoff"].score for t in traces}
     assert payoffs["player0"] + payoffs["player1"] == 0  # zero-sum
     assert abs(payoffs["player0"]) in (1.0, 2.0)
     for trace in traces:

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/tests/v1/test_judges.py

@@ -453,7 +453,7 @@ async def test_error_attribution(monkeypatch, tmp_path):
     # model failure: empty reply -> judge skipped, reward 0.0, NO error
     trace = make_trace(reply="")
     await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None)
-    assert trace.rewards["reference"] == 0.0
+    assert trace.rewards["reference"].score == 0.0
     # judge failure: unparseable verdict -> the rollout errors, no reward recorded
     trace = make_trace()
     with pytest.raises(vf.TaskError, match="no yes/no verdict"):
@@ -623,11 +623,13 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
     taskset = JudgedTaskset(cfg)
     trace = make_trace()
     await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None)
-    assert trace.rewards["own"] == 0.25  # decorated rewards still run
+    assert trace.rewards["own"].score == 0.25  # decorated rewards still run
+    assert trace.rewards["reference"] == vf.Reward(
+        score=1.0, weight=0.5
+    )  # raw score + weight, under the id-derived name
     assert (
-        trace.rewards["reference"] == 0.5
-    )  # 1.0 * weight 0.5, under the id-derived name
-    assert trace.rewards["quality"] == 0.75  # the rubric's aggregate, under its `name`
+        trace.rewards["quality"].score == 0.75
+    )  # the rubric's aggregate, under its `name`
     assert (
         len(trace.info["judge"]) == 2
     )  # every judge call recorded (rubric = one call)
@@ -636,4 +638,4 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
 async def test_task_without_judges_scores_as_before():
     trace = make_trace()
     await JudgedTask(trace.task.data).score(trace, runtime=None)
-    assert trace.rewards == {"own": 0.25}
+    assert trace.rewards == {"own": vf.Reward(score=0.25)}

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/verifiers/v1/__init__.py

@@ -107,6 +107,7 @@ from verifiers.v1.trace import (
     EvalRunInfo,
     GenerationSpan,
     ModelCall,
+    Reward,
     RunInfo,
     TimeSpan,
     TimeSplit,
@@ -171,6 +172,7 @@ __all__ = [
     "Trace",
     "TraceTask",
     "WireTrace",
+    "Reward",
     "Episode",
     "WireEpisode",
     "TRACE_VERSION",

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/verifiers/v1/cli/dashboard/eval.py

@@ -341,13 +341,19 @@ def _score_segments(traces: list[Trace], source: str) -> str | None:
         return None
     segments = []
     for name in names:
-        mean = format_mean(
-            traces, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0)
-        )
+        mean = format_mean(traces, lambda t, n=name, s=source: _score(t, s, n))
         segments.append(f"{name} {mean}")
     return "  ·  ".join(segments)
 
 
+def _score(trace: Trace, source: str, name: str) -> float:
+    """Rewards carry raw score + weight; the breakdown shows the raw score."""
+    if source == "rewards":
+        reward = trace.rewards.get(name)
+        return reward.score if reward is not None else 0.0
+    return trace.metrics.get(name, 0.0)
+
+
 def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     """Score rows read the policy view (`scored` — trainable traces); with several
     roles in play they split per role, each role averaging over its OWN traces (no

verifiers-0.2.2.dev27/verifiers/v1/envs/agentic_judge/__init__.py

@@ -0,0 +1,15 @@
+from verifiers.v1.envs.agentic_judge.env import (
+    AgenticJudgeEnv,
+    AgenticJudgeEnvConfig,
+    Criterion,
+    JudgeTaskConfig,
+    ScoreConfig,
+)
+
+__all__ = [
+    "AgenticJudgeEnv",
+    "AgenticJudgeEnvConfig",
+    "Criterion",
+    "JudgeTaskConfig",
+    "ScoreConfig",
+]

verifiers-0.2.2.dev27/verifiers/v1/envs/agentic_judge/env.py

@@ -0,0 +1,347 @@
+"""agentic-judge: a solver plays the task, a judge verifies it in the same box.
+
+A reusable env (`--env.id agentic-judge` over any taskset): the box is
+provisioned from the solver's runtime policy, the solver plays the task in it,
+and a code-executing judge then inspects the work as the agent left it, with
+the solver's full trace record uploaded at `/tmp/trace.json`. The judge grades
+rubric criteria (`[env.task]`: policy prompt, criteria file) and writes its
+verdicts to `/tmp/verdict.json`; `finalize()` validates them strictly onto the
+solver's trace — `judge/<name>` metrics plus a weighted-mean `judge` reward,
+composed with the taskset's own rewards via `[env.score]` (judge-only by
+default).
+"""
+
+import json
+import math
+import re
+import tomllib
+from pathlib import Path
+
+from pydantic import field_validator
+
+import verifiers.v1 as vf
+from verifiers.v1.types import StrictBaseModel
+
+VERDICT_FILE = "/tmp/verdict.json"
+TRACE_FILE = "/tmp/trace.json"
+
+GRADE_PROMPT = """\
+You are grading another agent's attempt at a task. Verify the work EMPIRICALLY:
+reconstruct what the agent did from its trace and test it with real execution
+in your sandbox — never take the trace's word for an outcome you can check."""
+
+TASK_SECTION = """\
+## The task the agent was given
+
+{prompt}"""
+
+
+class Criterion(StrictBaseModel):
+    """One rubric criterion — the plugged rubric judge's format, mirrored so the
+    same `criteria` files grade both judges."""
+
+    name: str
+    """Key for the criterion's metric (`judge/<name>`)."""
+    text: str
+    weight: float = 1.0
+    """The criterion's share of the reward."""
+    choices: list[str] = ["no", "yes"]
+    """Allowed answers, ordered **worst → best**: the first scores 0.0, the last 1.0, the rest
+    evenly spaced by rank. Default `["no", "yes"]` is a binary check. Needs >= 2, no duplicates."""
+
+    @field_validator("choices")
+    @classmethod
+    def _check_choices(cls, v: list[str]) -> list[str]:
+        if len(v) < 2:
+            raise ValueError(f"`choices` needs at least two options, got {v}")
+        if len(set(v)) != len(v):
+            raise ValueError(f"`choices` has duplicate options: {v}")
+        return v
+
+
+SOLVED = Criterion(
+    name="solved",
+    text="The task is fully solved: what the task asked for is achieved, and "
+    "you verified it with real execution.",
+)
+
+
+def _verdict_section(criteria: list[Criterion]) -> str:
+    listing = "\n".join(
+        f"- {c.name}: {c.text} (answer one of, worst to best: {', '.join(c.choices)})"
+        for c in criteria
+    )
+    return f"""\
+## Your verdict
+
+Grade the attempt on these criteria:
+
+{listing}
+
+When you are done verifying, write your verdict as JSON to `{VERDICT_FILE}`:
+
+    {{"verdicts": [{{"name": "<criterion name>", "reason": "<one sentence citing \
+what you verified>", "verdict": "<answer>"}}, ...]}}
+
+with one entry per criterion, using each criterion's exact name. For each, first
+write the one-sentence reason grounded in what you actually verified, then set
+verdict to exactly one of the options listed in parentheses after that
+criterion."""
+
+
+def _render(template: str, **fields: str) -> str:
+    """Substitute documented placeholders in one pass over the original template —
+    str.format would crash on any literal brace in a custom prompt, and sequential
+    replaces would re-scan substituted values. An unknown placeholder stays as written."""
+    pattern = re.compile(r"\{(" + "|".join(map(re.escape, fields)) + r")\}")
+    return pattern.sub(lambda m: fields[m.group(1)], template)
+
+
+SANDBOX_NOTE = f"""\
+## Your workspace
+
+Your sandbox is the SAME box the graded agent worked in, in the state the agent
+left it — its edits (and any scoring side effects) are applied. The agent's raw
+trace record (JSON: messages, tool calls, and its `info` artifacts) is uploaded
+at `{TRACE_FILE}`. The record can be very large — never dump it whole; peek
+selectively (list its keys, then slice out specific fields with python or jq)
+and pull only what you need. It is complete — it may also carry the task's own
+scores/metrics and reference material (a gold answer, a reference solution,
+held-out tests). Those are context, not your standard: recorded scores can be
+wrong and references can be narrower than the task; do not over-index on how a
+reference solves it. Your verdict is what YOU verified by execution."""
+
+HINT_SECTION = """\
+## Hints
+
+{hint}"""
+
+
+class JudgeTask(vf.Task):
+    """The judge's verdict task: the solver task's world mirrored onto the minted
+    row, the trace record written (and any stale verdict removed) before the
+    judge starts, verdict scraped off the live box after it exits. `NEEDS_CONTAINER`
+    keeps `Agent.run`'s per-task backstop aligned with the judge's declared need."""
+
+    NEEDS_CONTAINER = True
+
+    def __init__(self, data: vf.TaskData, files: dict[str, bytes]) -> None:
+        super().__init__(data)
+        self.files = files
+
+    @classmethod
+    def from_trace(cls, solution: vf.Trace, config: "JudgeTaskConfig") -> "JudgeTask":
+        """Mint the judge's task from the solver's finished trace."""
+        solved = solution.task.data
+        files = {TRACE_FILE: json.dumps(solution.to_record()).encode()}
+        template = config.build_prompt()
+        body = _render(template, prompt=solved.prompt_text)
+        if "{prompt}" not in template:
+            # A policy that doesn't place the task statement itself still needs it.
+            body += "\n\n" + _render(TASK_SECTION, prompt=solved.prompt_text)
+        sections = [body, _verdict_section(config.criteria()), SANDBOX_NOTE]
+        if (hint := config.build_hint()) is not None:
+            sections.insert(1, _render(HINT_SECTION, hint=hint))
+        prompt = "\n\n".join(sections)
+        return cls(
+            vf.TaskData(
+                idx=solved.idx,
+                prompt=prompt,
+                image=solved.image,
+                workdir=solved.workdir,
+                resources=solved.resources,
+            ),
+            files=files,
+        )
+
+    async def setup(self, trace: vf.Trace, runtime: vf.Runtime) -> None:
+        # The solver had this box first: a pre-seeded verdict must never read as
+        # the judge's own, and a file (or planted symlink) at an upload path must
+        # never survive it — a symlinked TRACE_FILE would redirect the write onto
+        # any file the solver chose.
+        await runtime.run(["rm", "-f", VERDICT_FILE, *self.files], env={})
+        for path, content in self.files.items():
+            await runtime.write(path, content)
+
+    async def finalize(self, trace: vf.Trace, runtime: vf.Runtime) -> None:
+        """Scrape the verdict off the box while it's alive. A judge that wrote no
+        file (or garbage) fails HERE — on the judge's own trace, the retryable
+        unit — never silently."""
+        try:
+            raw = await runtime.read(VERDICT_FILE)
+        except Exception as e:
+            raise ValueError(
+                f"the judge wrote no verdict to {VERDICT_FILE}; its final act must "
+                'be writing {"verdicts": [{"name", "reason", "verdict"}, ...]} there'
+            ) from e
+        trace.info["verdict"] = json.loads(raw)
+
+
+class JudgeTaskConfig(vf.BaseConfig):
+    """The judge's minted task: the grading policy and what lands in its box."""
+
+    prompt: Path | str | None = None
+    """Grading-policy override: inline text, or a policy file (a value ending in
+    `.md`/`.txt` is read from disk). Replaces only the policy body — the verdict
+    contract and workspace note are always appended, so a custom policy cannot
+    break verdict scraping. May reference `{prompt}` (the solver task's prompt);
+    if it doesn't, the task statement is appended after the policy."""
+    hint: Path | str | None = None
+    """Optional hints injected as their own section (inline text or a `.md`/
+    `.txt` file): task-family pointers into the trace or box — e.g. for math,
+    where the reference answer lives in the record; for SWE, to diff the repo
+    or read `info.patch`."""
+    rubric: Path | None = None
+    """Criteria the judge grades against: a `.toml`/`.json` file with a
+    `criteria` list — the plugged rubric judge's format, so the same rubric
+    files work for both. None grades the single built-in `solved` criterion."""
+
+    @staticmethod
+    def _resolve(value: Path | str) -> str:
+        path = Path(value)
+        if isinstance(value, Path) or path.suffix in (".md", ".txt"):
+            return path.read_text(encoding="utf-8")
+        return str(value)
+
+    def build_prompt(self) -> str:
+        if self.prompt is None:
+            return GRADE_PROMPT + "\n\n" + TASK_SECTION
+        return self._resolve(self.prompt)
+
+    def build_hint(self) -> str | None:
+        return self._resolve(self.hint) if self.hint is not None else None
+
+    def criteria(self) -> list[Criterion]:
+        if self.rubric is None:
+            return [SOLVED]
+        text = self.rubric.read_text(encoding="utf-8")
+        data = (
+            tomllib.loads(text)
+            if self.rubric.suffix.lower() == ".toml"
+            else json.loads(text)
+        )
+        items = data.get("criteria", []) if isinstance(data, dict) else data
+        criteria = [Criterion.model_validate(item) for item in items]
+        if not criteria:
+            raise ValueError(f"rubric file '{self.rubric}' lists no criteria")
+        names = [criterion.name for criterion in criteria]
+        if len(set(names)) != len(names):
+            raise ValueError(
+                f"rubric file '{self.rubric}' has duplicate criterion names"
+            )
+        if bad := [c.name for c in criteria if not 0 <= c.weight < math.inf]:
+            raise ValueError(
+                f"rubric '{self.rubric}' has negative or non-finite criterion "
+                f"weights: {bad}"
+            )
+        if sum(criterion.weight for criterion in criteria) <= 0:
+            raise ValueError(f"rubric '{self.rubric}' has no positive criterion weight")
+        return criteria
+
+
+class ScoreConfig(vf.BaseConfig):
+    """How the judge's verdict composes with the taskset's own rewards on the
+    solver's trace. Judge-only by default."""
+
+    task_weight: float = 0.0
+    """Scale applied to the taskset's own rewards; 1 keeps them next to the verdict."""
+    judge_weight: float = 1.0
+    """Weight of the judge's verdict in the solver's reward."""
+
+
+class AgenticJudgeEnvConfig(vf.EnvConfig):
+    solver: vf.AgentConfig = vf.AgentConfig()
+    """The solver agent. It owns the shared box, so its runtime must be a
+    container: `--env.solver.runtime.type docker|prime`."""
+    judge: vf.AgentConfig = vf.AgentConfig()
+    """The judge agent. It plays in the solver's box; its own runtime policy is
+    ignored (overwritten with the solver's)."""
+    task: JudgeTaskConfig = JudgeTaskConfig()
+    score: ScoreConfig = ScoreConfig()
+
+
+class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
+    def __init__(self, config: AgenticJudgeEnvConfig) -> None:
+        # The judge plays in the solver's box, so its effective runtime IS the
+        # solver's policy — aligning the config keeps the base env's subprocess
+        # warning and the runtime stamped on the judge's trace truthful.
+        config.judge = config.judge.model_copy(
+            update={"runtime": config.solver.runtime}
+        )
+        super().__init__(config)
+        self._check_agents()
+        # A missing policy file or a malformed rubric fails here, not mid-episode.
+        config.task.build_prompt()
+        config.task.build_hint()
+        config.task.criteria()
+
+    def _check_agents(self) -> None:
+        """The judge executes real code, never on the host — refuse an impossible
+        pairing at construction, not after burning a full solver run."""
+        judge = self._harnesses["judge"]
+        if not judge.EXECUTES_CODE:
+            raise ValueError(
+                "agentic-judge plays a code-executing judge in its own sandbox, but "
+                f"harness {judge.config.id!r} is a tool-less chat loop — a verdict "
+                "that needs no execution is a plugged judge "
+                "(--env.taskset.task.judges), not an agent."
+            )
+        if isinstance(self.config.solver.runtime, vf.SubprocessConfig):
+            raise ValueError(
+                "agentic-judge plays its judge in the solver's box, but the solver "
+                "(which provisions it) resolves to the subprocess runtime; use "
+                "--env.solver.runtime.type docker or prime"
+            )
+
+    async def setup(self, agents: vf.Agents) -> None:
+        # The judge grades the policy; its tokens are never training data.
+        agents.judge.trainable = False
+
+    async def run(self, task: vf.Task, agents: vf.Agents) -> None:
+        async with agents.solver.provision(task) as box:
+            solution = await agents.solver.run(task, runtime=box)
+            judge_task = JudgeTask.from_trace(solution, self.config.task)
+            await agents.judge.run(judge_task, runtime=box)
+
+    async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
+        by_agent = {t.agent_name: t for t in episode.traces}
+        solution, verdict = by_agent["solver"], by_agent["judge"]
+        data = verdict.info.get("verdict")
+        if not isinstance(data, dict) or not isinstance(data.get("verdicts"), list):
+            raise ValueError(
+                f"no verdicts on the judge's trace (expected {VERDICT_FILE} with a "
+                '"verdicts" list)'
+            )
+        criteria = self.config.task.criteria()
+        by_criterion = {c.name: c for c in criteria}
+        answers: dict[str, str] = {}
+        for entry in data["verdicts"]:
+            if not isinstance(entry, dict):
+                raise ValueError(f"verdict entry {entry!r} is not an object")
+            name = str(entry.get("name"))
+            if name in answers:
+                # Contradictory duplicates must not collapse to whichever came last.
+                raise ValueError(f"judge answered criterion {name!r} more than once")
+            answers[name] = str(entry.get("verdict"))
+        if sorted(answers) != sorted(by_criterion):
+            raise ValueError(
+                f"judge verdicts name {sorted(answers)}, expected the rubric's "
+                f"{sorted(by_criterion)}"
+            )
+        scores: dict[str, float] = {}
+        for name, answer in answers.items():
+            choices = by_criterion[name].choices
+            # An off-menu answer is a judge failure, not a zero score.
+            if answer not in choices:
+                raise ValueError(
+                    f"judge answered {answer!r} for '{name}', expected one of {choices}"
+                )
+            scores[name] = choices.index(answer) / (len(choices) - 1)
+        for criterion in criteria:
+            solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
+        if self.config.score.task_weight != 1.0:
+            for reward in solution.rewards.values():
+                reward.weight *= self.config.score.task_weight
+        total = sum(criterion.weight for criterion in criteria)
+        reward = sum(c.weight * scores[c.name] for c in criteria) / total
+        solution.record_reward("judge", reward, weight=self.config.score.judge_weight)

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/verifiers/v1/legacy.py

@@ -37,6 +37,7 @@ from verifiers.v1.trace import (
     Error,
     GenerationSpan,
     ModelCall,
+    Reward,
     TimeSpan,
     TimeSplit,
     Timing,
@@ -270,7 +271,7 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
             data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")),
         ),
         tools=_to_v1_tools(out.get("tool_defs")),
-        rewards={"reward": float(out.get("reward") or 0.0)},
+        rewards={"reward": Reward(score=float(out.get("reward") or 0.0))},
         metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
         info=dict(out.get("info") or {}),
         is_completed=bool(out.get("is_completed", True)),

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/verifiers/v1/push.py

@@ -90,9 +90,10 @@ def trace_to_sample(
         else None,
         "info": dict(trace.info) or None,
     }
-    # Flatten sub-rewards to top-level keys the way v0 does; env metrics stay nested.
-    for name, value in trace.rewards.items():
-        sample.setdefault(name, value)
+    # Flatten sub-rewards to top-level keys the way v0 does (raw scores, as v0's
+    # per-function outputs were); env metrics stay nested.
+    for name, reward in trace.rewards.items():
+        sample.setdefault(name, reward.score)
     return sample
 
 
@@ -127,7 +128,8 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     sums: dict[str, float] = {}
     counts: dict[str, int] = {}
     for trace in scored:
-        for name, value in {**trace.rewards, **trace.metrics}.items():
+        scores = {name: reward.score for name, reward in trace.rewards.items()}
+        for name, value in {**scores, **trace.metrics}.items():
             sums[name] = sums.get(name, 0.0) + value
             counts[name] = counts.get(name, 0) + 1
     n = len(scored)

{verifiers-0.2.2.dev25 → verifiers-0.2.2.dev27}/verifiers/v1/trace.py

@@ -266,7 +266,7 @@ _NODE_DUMP_EXCLUDE: dict = {
 """Raw tensor fields kept on the msgpack wire but excluded from JSON records."""
 
 
-TRACE_VERSION = 3
+TRACE_VERSION = 4
 """Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on
 breaking shape changes; optional-with-default fields are additive and don't bump it."""
 
@@ -343,6 +343,21 @@ class TraceTask(StrictBaseModel, Generic[DataT]):
     """The (immutable) row being solved."""
 
 
+class Reward(StrictBaseModel):
+    """One named reward as recorded on the trace: the raw score next to its weight,
+    so records keep both readable and the weighted sum stays a derived view."""
+
+    score: float
+    """The raw value the reward function returned, unweighted."""
+    weight: float = 1.0
+    """The multiplier `score` carries in the trace-level `reward` sum."""
+
+    @property
+    def value(self) -> float:
+        """This reward's weighted contribution to the trace-level `reward`."""
+        return self.score * self.weight
+
+
 class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
     id: str = Field(default_factory=lambda: uuid.uuid4().hex)
     """Unique id for this rollout, auto-generated per trace."""
@@ -369,8 +384,9 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
     """Every provider exchange behind the sampled turns, in order: raw wire request/response
     plus per-call timing and errors, linked into `nodes` via `ModelCall.node`."""
 
-    rewards: dict[str, float] = Field(default_factory=dict)
-    """Weighted contributions from task rewards, judges, and the env's `score()`."""
+    rewards: dict[str, Reward] = Field(default_factory=dict)
+    """Named rewards from tasks, judges, and the env's `score()` — each keeps its
+    raw `score` and `weight`; the trace-level `reward` is their weighted sum."""
     metrics: dict[str, float] = Field(default_factory=dict)
     """Unweighted metrics from tasks, harnesses, and judges."""
     info: dict[str, Any] = Field(default_factory=dict)
@@ -400,7 +416,7 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     @property
     def reward(self) -> float:
-        return sum(self.rewards.values())
+        return sum(r.value for r in self.rewards.values())
 
     @property
     def error(self) -> Error | None:
@@ -559,12 +575,12 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
             self.extra_usage.append(response.usage)
 
     def record_reward(self, name: str, value: float, weight: float = 1.0) -> None:
-        contribution = float(value) * float(weight)
+        reward = Reward(score=float(value), weight=float(weight))
         if name in self.rewards:
             logger.warning(
-                "reward %r overridden: %s -> %s", name, self.rewards[name], contribution
+                "reward %r overridden: %s -> %s", name, self.rewards[name], reward
             )
-        self.rewards[name] = contribution
+        self.rewards[name] = reward
 
     def stamp(self, run: RunInfo | None = None, **info: Any) -> None:
         """Stamp identity only the consumer knows (the eval CLI / a trainer) onto the

verifiers-0.2.2.dev25/verifiers/v1/envs/agentic_judge/__init__.py

@@ -1,3 +0,0 @@
-from verifiers.v1.envs.agentic_judge.env import AgenticJudgeEnv, AgenticJudgeEnvConfig
-
-__all__ = ["AgenticJudgeEnv", "AgenticJudgeEnvConfig"]