verifiers 0.2.2.dev26__py3-none-any.whl → 0.2.2.dev28__py3-none-any.whl

verifiers/v1/__init__.py

@@ -107,6 +107,7 @@ from verifiers.v1.trace import (
     EvalRunInfo,
     GenerationSpan,
     ModelCall,
+    Reward,
     RunInfo,
     TimeSpan,
     TimeSplit,
@@ -171,6 +172,7 @@ __all__ = [
     "Trace",
     "TraceTask",
     "WireTrace",
+    "Reward",
     "Episode",
     "WireEpisode",
     "TRACE_VERSION",

verifiers/v1/cli/dashboard/eval.py

@@ -341,13 +341,19 @@ def _score_segments(traces: list[Trace], source: str) -> str | None:
         return None
     segments = []
     for name in names:
-        mean = format_mean(
-            traces, lambda t, n=name, s=source: getattr(t, s).get(n, 0.0)
-        )
+        mean = format_mean(traces, lambda t, n=name, s=source: _score(t, s, n))
         segments.append(f"{name} {mean}")
     return "  ·  ".join(segments)
 
 
+def _score(trace: Trace, source: str, name: str) -> float:
+    """Rewards carry raw score + weight; the breakdown shows the raw score."""
+    if source == "rewards":
+        reward = trace.rewards.get(name)
+        return reward.score if reward is not None else 0.0
+    return trace.metrics.get(name, 0.0)
+
+
 def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     """Score rows read the policy view (`scored` — trainable traces); with several
     roles in play they split per role, each role averaging over its OWN traces (no

verifiers/v1/envs/agentic_judge/env.py

@@ -340,8 +340,8 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
         for criterion in criteria:
             solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
         if self.config.score.task_weight != 1.0:
-            for name in solution.rewards:
-                solution.rewards[name] *= self.config.score.task_weight
+            for reward in solution.rewards.values():
+                reward.weight *= self.config.score.task_weight
         total = sum(criterion.weight for criterion in criteria)
         reward = sum(c.weight * scores[c.name] for c in criteria) / total
         solution.record_reward("judge", reward, weight=self.config.score.judge_weight)

verifiers/v1/legacy.py

@@ -37,6 +37,7 @@ from verifiers.v1.trace import (
     Error,
     GenerationSpan,
     ModelCall,
+    Reward,
     TimeSpan,
     TimeSplit,
     Timing,
@@ -270,7 +271,7 @@ def rollout_output_to_trace(out: dict, task_idx: int) -> Trace:
             data=_to_wire_task(task_idx, out.get("prompt"), out.get("answer")),
         ),
         tools=_to_v1_tools(out.get("tool_defs")),
-        rewards={"reward": float(out.get("reward") or 0.0)},
+        rewards={"reward": Reward(score=float(out.get("reward") or 0.0))},
         metrics={k: float(v) for k, v in (out.get("metrics") or {}).items()},
         info=dict(out.get("info") or {}),
         is_completed=bool(out.get("is_completed", True)),

verifiers/v1/push.py

@@ -90,9 +90,10 @@ def trace_to_sample(
         else None,
         "info": dict(trace.info) or None,
     }
-    # Flatten sub-rewards to top-level keys the way v0 does; env metrics stay nested.
-    for name, value in trace.rewards.items():
-        sample.setdefault(name, value)
+    # Flatten sub-rewards to top-level keys the way v0 does (raw scores, as v0's
+    # per-function outputs were); env metrics stay nested.
+    for name, reward in trace.rewards.items():
+        sample.setdefault(name, reward.score)
     return sample
 
 
@@ -127,7 +128,8 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     sums: dict[str, float] = {}
     counts: dict[str, int] = {}
     for trace in scored:
-        for name, value in {**trace.rewards, **trace.metrics}.items():
+        scores = {name: reward.score for name, reward in trace.rewards.items()}
+        for name, value in {**scores, **trace.metrics}.items():
             sums[name] = sums.get(name, 0.0) + value
             counts[name] = counts.get(name, 0) + 1
     n = len(scored)

verifiers/v1/trace.py

@@ -266,7 +266,7 @@ _NODE_DUMP_EXCLUDE: dict = {
 """Raw tensor fields kept on the msgpack wire but excluded from JSON records."""
 
 
-TRACE_VERSION = 3
+TRACE_VERSION = 4
 """Version of the trace record schema (see `Trace.model_json_schema()`). Bumped on
 breaking shape changes; optional-with-default fields are additive and don't bump it."""
 
@@ -343,6 +343,21 @@ class TraceTask(StrictBaseModel, Generic[DataT]):
     """The (immutable) row being solved."""
 
 
+class Reward(StrictBaseModel):
+    """One named reward as recorded on the trace: the raw score next to its weight,
+    so records keep both readable and the weighted sum stays a derived view."""
+
+    score: float
+    """The raw value the reward function returned, unweighted."""
+    weight: float = 1.0
+    """The multiplier `score` carries in the trace-level `reward` sum."""
+
+    @property
+    def value(self) -> float:
+        """This reward's weighted contribution to the trace-level `reward`."""
+        return self.score * self.weight
+
+
 class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
     id: str = Field(default_factory=lambda: uuid.uuid4().hex)
     """Unique id for this rollout, auto-generated per trace."""
@@ -369,8 +384,9 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
     """Every provider exchange behind the sampled turns, in order: raw wire request/response
     plus per-call timing and errors, linked into `nodes` via `ModelCall.node`."""
 
-    rewards: dict[str, float] = Field(default_factory=dict)
-    """Weighted contributions from task rewards, judges, and the env's `score()`."""
+    rewards: dict[str, Reward] = Field(default_factory=dict)
+    """Named rewards from tasks, judges, and the env's `score()` — each keeps its
+    raw `score` and `weight`; the trace-level `reward` is their weighted sum."""
     metrics: dict[str, float] = Field(default_factory=dict)
     """Unweighted metrics from tasks, harnesses, and judges."""
     info: dict[str, Any] = Field(default_factory=dict)
@@ -400,7 +416,7 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     @property
     def reward(self) -> float:
-        return sum(self.rewards.values())
+        return sum(r.value for r in self.rewards.values())
 
     @property
     def error(self) -> Error | None:
@@ -559,12 +575,12 @@ class Trace(StrictBaseModel, Generic[DataT, StateT, AgentConfigT]):
             self.extra_usage.append(response.usage)
 
     def record_reward(self, name: str, value: float, weight: float = 1.0) -> None:
-        contribution = float(value) * float(weight)
+        reward = Reward(score=float(value), weight=float(weight))
         if name in self.rewards:
             logger.warning(
-                "reward %r overridden: %s -> %s", name, self.rewards[name], contribution
+                "reward %r overridden: %s -> %s", name, self.rewards[name], reward
             )
-        self.rewards[name] = contribution
+        self.rewards[name] = reward
 
     def stamp(self, run: RunInfo | None = None, **info: Any) -> None:
         """Stamp identity only the consumer knows (the eval CLI / a trainer) onto the

{verifiers-0.2.2.dev26.dist-info → verifiers-0.2.2.dev28.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev26
+Version: 0.2.2.dev28
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev26.dist-info → verifiers-0.2.2.dev28.dist-info}/RECORD

@@ -167,7 +167,7 @@ verifiers/utils/threaded_sandbox_client.py,sha256=Pbr8MA4FDPEitL6z88S8T1qLJOmtXN
 verifiers/utils/tool_utils.py,sha256=gInWZODWQUZN4TyEMuuITyOUm9qlHJxtcvr1zZl_zJs,1020
 verifiers/utils/usage_utils.py,sha256=GPLC0xGY_Obrr8X7huWY-2iODZ7tgX-MtO8WSDN4rXI,3904
 verifiers/utils/version_utils.py,sha256=am3hZLnlaUWTFdllGZR1VMN91hEhiiE8FDmPf1cOG1k,2642
-verifiers/v1/__init__.py,sha256=29oq_Ng33Shgo2x7OGciV7SQrN0ndB8eBxsbebCggqw,6833
+verifiers/v1/__init__.py,sha256=z0yUgI1iVf8znsAtBG2fpRKuwipaQ5yS4PvGhWJbKtE,6859
 verifiers/v1/agent.py,sha256=FP38GsZXuEJe3tJ-U2THpwBf9NWQeo847hWRZoeggNw,31956
 verifiers/v1/decorators.py,sha256=XRMkUQSyvXCYP5fOwzBYV5qEOlxLg6rzYhnhrHHHTIQ,3838
 verifiers/v1/env.py,sha256=w6sHWdWijLRZzhwVyJjadnGtziNqJAPCeMUoNdzHoLg,17712
@@ -176,9 +176,9 @@ verifiers/v1/errors.py,sha256=Pj5Om8x1fP3TDPQQn6nUCoSPoZRsy2JeBz8pXhpPrDY,6883
 verifiers/v1/graph.py,sha256=Mivq5jICUcyK-fhomFTwP4fQn688MXg6-hmdf03aC4Y,27703
 verifiers/v1/harness.py,sha256=Ukzhk7wxSUSVnN4tRAUMfniN9R9MBSNzs2SiT5YcRzQ,10711
 verifiers/v1/judge.py,sha256=ZWvr6uCniSwqzzjKrlyh-lzfCh5q-VyB16W42x0aWlg,9449
-verifiers/v1/legacy.py,sha256=b53EDDOImu8rSaZR_ZE7jNaQTJ6aIw5T8hxlXU4fTh0,22280
+verifiers/v1/legacy.py,sha256=8eVGhutQEgJG4qabhph4Xs1VzTWvmMxMn3-Zk-6ViWM,22306
 verifiers/v1/loaders.py,sha256=FdICekH_c9WYkUe0XJCI9AWxL1sJjjzzJPJ7ZJdeyRk,9962
-verifiers/v1/push.py,sha256=N8UNz0eHKVH_vgh6ZuN-efqPOgYtscQBzn8zz-JYmYw,11138
+verifiers/v1/push.py,sha256=VfLESKAlL8WXzfXx6CrqwMJ-CvPTkP-tUf6Rin12V8w,11273
 verifiers/v1/retries.py,sha256=ZQxY6R_FoXooERmhIMZnhY3b2koJOVjaIvkSO9gw53E,5379
 verifiers/v1/rollout.py,sha256=8_LC938Ws9uWFRPz2DDpivttqDALb3Yir7J5wjnsoIE,20332
 verifiers/v1/scoring.py,sha256=I_mtqhTZ195hj2-CB5jzr3BY5GFKgNEvTf331f7Is4k,5740
@@ -186,7 +186,7 @@ verifiers/v1/session.py,sha256=J6ZPuvsg6vMTJAs9dMUMtfdjCuvp6Hom877DVdKFKlY,6912
 verifiers/v1/state.py,sha256=R8tyQv8nsFV2pztrquDqCOa1Mk1fAp3w2GjqUugZwE0,689
 verifiers/v1/task.py,sha256=qKAAF154ykgvrms2Y2Or6kgGVPvBMsFCD6KqX2QOQaU,10912
 verifiers/v1/taskset.py,sha256=PX1-skAVhSpGF07qjxcG0Ctmq1LiMftMxxok-uc8qRY,3939
-verifiers/v1/trace.py,sha256=J_g3DgruE8B747fjkmhiI9UE8KIesf9mqAANMNo6Z1E,25976
+verifiers/v1/trace.py,sha256=OE7vW6sYGjA18JdRJuHM9UiRaq9XkGS0Yv2m7nuvEYs,26619
 verifiers/v1/types.py,sha256=5tZyG4r4bLJ9a14oybHy7bSCpt5X5A17eRwmIszuRo8,9118
 verifiers/v1/acp/__init__.py,sha256=9dwH6fLopNndRmcpRSYC8ghzzMquaLfgh645QM-Dwic,2189
 verifiers/v1/acp/_runner.py,sha256=FqEnHR0Q_YrCt_EuaJqiqTpd2_kFa41FqcEunwOhvTI,6856
@@ -201,7 +201,7 @@ verifiers/v1/cli/serve.py,sha256=VHzcr2bM8R3XoGEu6WvY1NSQoiBZsk49cVW5AB9N4Kg,266
 verifiers/v1/cli/validate.py,sha256=r3ByzcLc_DFkvlCB7NanYRnBSOIi5EU9cUANy74l3T0,9547
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
-verifiers/v1/cli/dashboard/eval.py,sha256=F--kqnwjeihY1IrQ55liMwpQJSIkcp1RZmZzhKLhXVY,34081
+verifiers/v1/cli/dashboard/eval.py,sha256=nmaWIq-4ormHZWjwlWm1QSFo383Sc4VFEkw9rYnpi00,34357
 verifiers/v1/cli/dashboard/replay.py,sha256=CvRVaf0dUum5v0IzPQbFPPikBQmTrkMeaunGfVMTWHc,2755
 verifiers/v1/cli/dashboard/validate.py,sha256=rLsQ_31DjTIZpC_VO7zSG41ncUEBxqCx8mL9nTnGiMc,3650
 verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -236,7 +236,7 @@ verifiers/v1/dialects/chat.py,sha256=DFvjmIW86jZUMWuz-hOqzxC6tiiFibiAzNEJ3JkpvjU
 verifiers/v1/dialects/responses.py,sha256=vVgeT7-aWjtfED2IKlDwTATKn_AQZruHNL6mkFTTZTU,13548
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=X7vQbbbfmJ_j-mJEfBkuB4a8QUPkUYkpG7yp0xYOEmg,279
-verifiers/v1/envs/agentic_judge/env.py,sha256=yd1hqZphIsJD8Z9eDnBU9aGdXmDaAu91QvcyFWaYy8M,15305
+verifiers/v1/envs/agentic_judge/env.py,sha256=_zEKG9i3LsOVIQWULXYAHpbmrQNPWBRayJx2yjvtPbQ,15307
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZnLMje6eVgyz5U,138
@@ -327,8 +327,8 @@ verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/sampling.py,sha256=JczGzBn6s3wsIrSY2Hy3hE8m6NreNgsNzhSjDikQqX0,1037
 verifiers/v1/utils/version.py,sha256=75ZtI2NHBmlb52KpcKLUpiSXp8q4bASr7uKeXoCKlT8,1582
-verifiers-0.2.2.dev26.dist-info/METADATA,sha256=MIaPrTSbtXpw4KSgSjhcqHTnlGiEKPhyLiVTphUTx10,4540
-verifiers-0.2.2.dev26.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev26.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev26.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev26.dist-info/RECORD,,
+verifiers-0.2.2.dev28.dist-info/METADATA,sha256=HaWApbU7oXTVKEbSZ0uIB1uTDVG2dBCkpHUDUIwp_04,4540
+verifiers-0.2.2.dev28.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev28.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev28.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev28.dist-info/RECORD,,