verifiers 0.2.2.dev26__tar.gz → 0.2.2.dev28__tar.gz

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev26
+Version: 0.2.2.dev28
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/tests/v1/test_e2e.py

@@ -197,7 +197,7 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
     )
     assert trace.ok, trace.errors
     assert trace.stop_condition == "user_closed"
-    assert trace.rewards["resumed"] == 1.0
+    assert trace.rewards["resumed"].score == 1.0
     assert trace.tools  # ACP-native tools, or Pi's MCP adapter meta-tool
     segments = trace.info["acp_segments"]
     assert len(segments) == 2
@@ -320,7 +320,7 @@ async def test_rubric_judge(run_v1, tmp_path):
         max_turns=2,
     )
     assert trace.ok
-    assert trace.rewards["rubric"] > 0  # the judge's verdict landed in the reward
+    assert trace.rewards["rubric"].score > 0  # the judge's verdict landed in the reward
     assert trace.metrics["rubric/always_yes"] == 1.0
     assert trace.info["judge"]  # the call was recorded onto the trace
 
@@ -428,9 +428,11 @@ async def test_env_id_agentic_judge(run_v1, tmp_path):
     (judge,) = [t for t in traces if t.agent_name == "judge"]
     assert solver.ok and judge.ok
     assert judge.trainable is False
-    assert solver.rewards["echoed"] == 0.5  # the task's own reward, rescaled
+    # The task's own reward keeps its raw score; the rescale lands on the weight.
+    assert solver.rewards["echoed"].score == 1.0
+    assert solver.rewards["echoed"].weight == 0.5
     assert isinstance(judge.info.get("verdict"), dict)  # scraped off the box
-    assert 0.0 <= solver.rewards["judge"] <= 1.0
+    assert 0.0 <= solver.rewards["judge"].score <= 1.0
 
 
 @pytest.mark.e2e
@@ -491,7 +493,7 @@ async def test_env_id_user_sim_with_tools(run_v1, tmp_path):
     assert assistant.ok and user.ok
     assert assistant.task.data.prompt is None  # the scenario stayed off the wire
     assert user.num_turns >= 1  # the modeled user actually drove the exchange
-    assert assistant.rewards["echoed"] == 1.0  # the tool ran, mid-conversation
+    assert assistant.rewards["echoed"].score == 1.0  # the tool ran, mid-conversation
     # The tool was advertised to the masked chat exactly as to any run.
     assert assistant.tools is not None
     assert any(tool.name == "echo_back" for tool in assistant.tools)
@@ -513,7 +515,7 @@ async def test_kuhn_poker_self_play(run_v1, tmp_path):
         rollout_timeout=300,
     )
     assert sorted(t.agent_name for t in traces) == ["player0", "player1"]
-    payoffs = {t.agent_name: t.rewards["payoff"] for t in traces}
+    payoffs = {t.agent_name: t.rewards["payoff"].score for t in traces}
     assert payoffs["player0"] + payoffs["player1"] == 0  # zero-sum
     assert abs(payoffs["player0"]) in (1.0, 2.0)
     for trace in traces:

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/tests/v1/test_judges.py

@@ -453,7 +453,7 @@ async def test_error_attribution(monkeypatch, tmp_path):
     # model failure: empty reply -> judge skipped, reward 0.0, NO error
     trace = make_trace(reply="")
     await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None)
-    assert trace.rewards["reference"] == 0.0
+    assert trace.rewards["reference"].score == 0.0
     # judge failure: unparseable verdict -> the rollout errors, no reward recorded
     trace = make_trace()
     with pytest.raises(vf.TaskError, match="no yes/no verdict"):
@@ -623,11 +623,13 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
     taskset = JudgedTaskset(cfg)
     trace = make_trace()
     await JudgedTask(trace.task.data, taskset.config.task).score(trace, runtime=None)
-    assert trace.rewards["own"] == 0.25  # decorated rewards still run
+    assert trace.rewards["own"].score == 0.25  # decorated rewards still run
+    assert trace.rewards["reference"] == vf.Reward(
+        score=1.0, weight=0.5
+    )  # raw score + weight, under the id-derived name
     assert (
-        trace.rewards["reference"] == 0.5
-    )  # 1.0 * weight 0.5, under the id-derived name
-    assert trace.rewards["quality"] == 0.75  # the rubric's aggregate, under its `name`
+        trace.rewards["quality"].score == 0.75
+    )  # the rubric's aggregate, under its `name`
     assert (
         len(trace.info["judge"]) == 2
     )  # every judge call recorded (rubric = one call)
@@ -636,4 +638,4 @@ async def test_task_score_runs_plugged_judges(tmp_path, fake_judge_model):
 async def test_task_without_judges_scores_as_before():
     trace = make_trace()
     await JudgedTask(trace.task.data).score(trace, runtime=None)
-    assert trace.rewards == {"own": 0.25}
+    assert trace.rewards == {"own": vf.Reward(score=0.25)}

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/verifiers/v1/__init__.py

@@ -107,6 +107,7 @@ from verifiers.v1.trace import (
     EvalRunInfo,
     GenerationSpan,
     ModelCall,
+    Reward,
     RunInfo,
     TimeSpan,
     TimeSplit,
@@ -171,6 +172,7 @@ __all__ = [
     "Trace",
     "TraceTask",
     "WireTrace",
+    "Reward",
     "Episode",
     "WireEpisode",
     "TRACE_VERSION",

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/verifiers/v1/cli/dashboard/eval.py

@@ -341,13 +341,19 @@ def _score_segments(traces: list[Trace], source: str) -> str | None:
         return None
     segments = []
     for name in names:
-        mean = format_mean(
-            traces, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0)
-        )
+        mean = format_mean(traces, lambda t, n=name, s=source: _score(t, s, n))
         segments.append(f"{name} {mean}")
     return "  ·  ".join(segments)
 
 
+def _score(trace: Trace, source: str, name: str) -> float:
+    """Rewards carry raw score + weight; the breakdown shows the raw score."""
+    if source == "rewards":
+        reward = trace.rewards.get(name)
+        return reward.score if reward is not None else 0.0
+    return trace.metrics.get(name, 0.0)
+
+
 def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     """Score rows read the policy view (`scored` — trainable traces); with several
     roles in play they split per role, each role averaging over its OWN traces (no

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/verifiers/v1/envs/agentic_judge/env.py

@@ -340,8 +340,8 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         for criterion in criteria:
             solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
         if self.config.score.task_weight != 1.0:
-            for name in solution.rewards:
-                solution.rewards[name] *= self.config.score.task_weight
+            for reward in solution.rewards.values():
+                reward.weight *= self.config.score.task_weight
         total = sum(criterion.weight for criterion in criteria)
         reward = sum(c.weight * scores[c.name] for c in criteria) / total
         solution.record_reward("judge", reward, weight=self.config.score.judge_weight)

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/verifiers/v1/legacy.py

@@ -37,6 +37,7 @@ from verifiers.v1.trace import (
     Error,
     GenerationSpan,
     ModelCall,
+    Reward,
     TimeSpan,
     TimeSplit,
     Timing,
@@ -270,7 +271,7 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
             data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")),
         ),
         tools=_to_v1_tools(out.get("tool_defs")),
-        rewards={"reward": float(out.get("reward") or 0.0)},
+        rewards={"reward": Reward(score=float(out.get("reward") or 0.0))},
         metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
         info=dict(out.get("info") or {}),
         is_completed=bool(out.get("is_completed", True)),

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/verifiers/v1/push.py

@@ -90,9 +90,10 @@ def trace_to_sample(
         else None,
         "info": dict(trace.info) or None,
     }
-    # Flatten sub-rewards to top-level keys the way v0 does; env metrics stay nested.
-    for name, value in trace.rewards.items():
-        sample.setdefault(name, value)
+    # Flatten sub-rewards to top-level keys the way v0 does (raw scores, as v0's
+    # per-function outputs were); env metrics stay nested.
+    for name, reward in trace.rewards.items():
+        sample.setdefault(name, reward.score)
     return sample
 
 
@@ -127,7 +128,8 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     sums: dict[str, float] = {}
     counts: dict[str, int] = {}
     for trace in scored:
-        for name, value in {**trace.rewards, **trace.metrics}.items():
+        scores = {name: reward.score for name, reward in trace.rewards.items()}
+        for name, value in {**scores, **trace.metrics}.items():
             sums[name] = sums.get(name, 0.0) + value
             counts[name] = counts.get(name, 0) + 1
     n = len(scored)

{verifiers-0.2.2.dev26 → verifiers-0.2.2.dev28}/verifiers/v1/trace.py

@@ -266,7 +266,7 @@ _NODE_DUMP_EXCLUDE: dict = {
 """Raw tensor fields kept on the msgpack wire but excluded from JSON records."""
 
 
-TRACE_VERSION = 3
+TRACE_VERSION = 4
 """Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on
 breaking shape changes; optional-with-default fields are additive and don't bump it."""
 
@@ -343,6 +343,21 @@ class TraceTask(StrictBaseModel, Generic[DataT]):
     """The (immutable) row being solved."""
 
 
+class Reward(StrictBaseModel):
+    """One named reward as recorded on the trace: the raw score next to its weight,
+    so records keep both readable and the weighted sum stays a derived view."""
+
+    score: float
+    """The raw value the reward function returned, unweighted."""
+    weight: float = 1.0
+    """The multiplier `score` carries in the trace-level `reward` sum."""
+
+    @property
+    def value(self) -> float:
+        """This reward's weighted contribution to the trace-level `reward`."""
+        return self.score * self.weight
+
+
 class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
     id: str = Field(default_factory=lambda: uuid.uuid4().hex)
     """Unique id for this rollout, auto-generated per trace."""
@@ -369,8 +384,9 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
     """Every provider exchange behind the sampled turns, in order: raw wire request/response
     plus per-call timing and errors, linked into `nodes` via `ModelCall.node`."""
 
-    rewards: dict[str, float] = Field(default_factory=dict)
-    """Weighted contributions from task rewards, judges, and the env's `score()`."""
+    rewards: dict[str, Reward] = Field(default_factory=dict)
+    """Named rewards from tasks, judges, and the env's `score()` — each keeps its
+    raw `score` and `weight`; the trace-level `reward` is their weighted sum."""
     metrics: dict[str, float] = Field(default_factory=dict)
     """Unweighted metrics from tasks, harnesses, and judges."""
     info: dict[str, Any] = Field(default_factory=dict)
@@ -400,7 +416,7 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     @property
     def reward(self) -> float:
-        return sum(self.rewards.values())
+        return sum(r.value for r in self.rewards.values())
 
     @property
     def error(self) -> Error | None:
@@ -559,12 +575,12 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
             self.extra_usage.append(response.usage)
 
     def record_reward(self, name: str, value: float, weight: float = 1.0) -> None:
-        contribution = float(value) * float(weight)
+        reward = Reward(score=float(value), weight=float(weight))
         if name in self.rewards:
             logger.warning(
-                "reward %r overridden: %s -> %s", name, self.rewards[name], contribution
+                "reward %r overridden: %s -> %s", name, self.rewards[name], reward
             )
-        self.rewards[name] = contribution
+        self.rewards[name] = reward
 
     def stamp(self, run: RunInfo | None = None, **info: Any) -> None:
         """Stamp identity only the consumer knows (the eval CLI / a trainer) onto the