verifiers 0.2.2.dev73__py3-none-any.whl → 0.2.2.dev74__py3-none-any.whl

verifiers/v1/cli/dashboard/eval.py

@@ -332,8 +332,8 @@ def Progress(
 
 def _score_segments(traces: list[Trace], source: str) -> str | None:
     """`name mean` segments for every reward/metric key seen across `traces`,
-    first-seen order (a trace records only the functions that ran for it, so keys
-    can vary); None when no trace recorded anything."""
+    first-seen order (a trace carries only its own task's signals — unscored ones
+    seeded as `None` — so keys can vary); None when no trace recorded anything."""
     names: list[str] = []
     for trace in traces:
         names.extend(n for n in getattr(trace, source) if n not in names)
@@ -347,11 +347,13 @@ def _score_segments(traces: list[Trace], source: str) -> str | None:
 
 
 def _score(trace: Trace, source: str, name: str) -> float:
-    """Rewards carry raw score + weight; the breakdown shows the raw score."""
+    """Rewards carry raw score + weight; the breakdown shows the raw score. A missing
+    or unscored (seeded `None`) entry reads as 0.0."""
     if source == "rewards":
         reward = trace.rewards.get(name)
         return reward.score if reward is not None else 0.0
-    return trace.metrics.get(name, 0.0)
+    value = trace.metrics.get(name)
+    return value if value is not None else 0.0
 
 
 def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:

verifiers/v1/envs/agentic_judge/env.py

@@ -398,7 +398,8 @@ class AgenticJudgeEnv(vf.Env[AgenticJudgeEnvConfig]):
             solution.record_metric(f"judge/{criterion.name}", scores[criterion.name])
         if self.config.score.task_weight != 1.0:
             for reward in solution.rewards.values():
-                reward.weight *= self.config.score.task_weight
+                if reward is not None:
+                    reward.weight *= self.config.score.task_weight
         total = sum(criterion.weight for criterion in criteria)
         reward = sum(c.weight * scores[c.name] for c in criteria) / total
         solution.record_reward("judge", reward, weight=self.config.score.judge_weight)

verifiers/v1/harness.py

@@ -12,7 +12,12 @@ from verifiers.v1.errors import HarnessError, SandboxError, boundary
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.task import TaskData
 from verifiers.v1.types import Messages
-from verifiers.v1.utils.decorators import discover_decorated, invoke_all
+from verifiers.v1.utils.decorators import (
+    discover_decorated,
+    invoke_all,
+    seed,
+    unseed,
+)
 
 if TYPE_CHECKING:
     # Annotation-only: `Trace` appears in signatures only, so this module stays
@@ -155,10 +160,12 @@ class Harness(ABC, Generic[ConfigT]):
         `runtime`) and can read what the harness left behind in the runtime."""
         available = {"task": trace.task.data, "trace": trace, "runtime": runtime}
         fns = discover_decorated(self, "metric")
+        seed(trace.metrics, (fn.__name__ for fn in fns))
         async with boundary(HarnessError, f"harness {self.config.id!r} metric"):
             results = await invoke_all(fns, available)
         for fn, result in zip(fns, results):
             if isinstance(result, Mapping):
+                unseed(trace.metrics, fn.__name__)
                 trace.record_metrics(result)
             else:
                 trace.record_metric(fn.__name__, result)

verifiers/v1/task.py

@@ -23,7 +23,12 @@ from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.state import StateT
 from verifiers.v1.types import Messages, content_text
 from verifiers.v1.utils.artifacts import Artifact
-from verifiers.v1.utils.decorators import discover_decorated, invoke_all
+from verifiers.v1.utils.decorators import (
+    discover_decorated,
+    invoke_all,
+    seed,
+    unseed,
+)
 from verifiers.v1.utils.generic import concrete_type
 
 if TYPE_CHECKING:
@@ -180,31 +185,36 @@ class Task(Generic[DataT, StateT, ConfigT]):
                     judge for judge in judges if not requires_runtime(judge.score)
                 ]
 
+            seed(trace.metrics, (fn.__name__ for fn in metrics))
+            seed(trace.rewards, (fn.__name__ for fn in rewards))
+            seed(trace.rewards, (judge.reward_name for judge in judges))
+
             metric_results = await invoke_all(metrics, available)
             for fn, result in zip(metrics, metric_results):
                 if isinstance(result, Mapping):
+                    unseed(trace.metrics, fn.__name__)
                     trace.record_metrics(result)
                 else:
                     trace.record_metric(fn.__name__, result)
             reward_results = await invoke_all(rewards, available)
             for fn, result in zip(rewards, reward_results):
                 weight = getattr(fn, "_vf_weight", 1.0)
-                items = (
-                    result.items()
-                    if isinstance(result, Mapping)
-                    else [(fn.__name__, result)]
-                )
+                if isinstance(result, Mapping):
+                    unseed(trace.rewards, fn.__name__)
+                    items = result.items()
+                else:
+                    items = [(fn.__name__, result)]
                 for key, value in items:
                     trace.record_reward(key, value, weight)
             judge_results = await invoke_all(
                 [judge.score for judge in judges], available
             )
             for judge, result in zip(judges, judge_results):
-                items = (
-                    result.items()
-                    if isinstance(result, Mapping)
-                    else [(judge.reward_name, result)]
-                )
+                if isinstance(result, Mapping):
+                    unseed(trace.rewards, judge.reward_name)
+                    items = result.items()
+                else:
+                    items = [(judge.reward_name, result)]
                 for key, value in items:
                     trace.record_reward(key, value, judge.config.weight)
 

verifiers/v1/trace.py

@@ -319,10 +319,11 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
     calls: list[ModelCall] = Field(default_factory=list)
     """Every model call; automatically recorded at intercept time + linked into `nodes`."""
 
-    rewards: dict[str, Reward] = Field(default_factory=dict)
-    """Named, weighted rewards"""
-    metrics: dict[str, float] = Field(default_factory=dict)
-    """Unweighted, named metrics"""
+    rewards: dict[str, Reward | None] = Field(default_factory=dict)
+    """Named, weighted rewards; `None` means scoring didn't run (e.g. because of a
+    preceding error)."""
+    metrics: dict[str, float | None] = Field(default_factory=dict)
+    """Unweighted, named metrics; `None` as in `rewards`."""
     info: dict[str, Any] = Field(default_factory=dict)
     """Scratch space for task-specific metadata."""
     state: StateT = Field(default_factory=State, exclude=True)
@@ -346,7 +347,7 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]):
 
     @property
     def reward(self) -> float:
-        return sum(r.value for r in self.rewards.values())
+        return sum(r.value for r in self.rewards.values() if r is not None)
 
     @property
     def has_error(self) -> bool:

verifiers/v1/utils/decorators.py

@@ -2,7 +2,7 @@
 
 import asyncio
 import inspect
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from typing import Any, TypeVar, overload
 
 F = TypeVar("F", bound=Callable[..., Any])
@@ -38,6 +38,22 @@ async def invoke_all(
     return await asyncio.gather(*(invoke(fn, available) for fn in fns))
 
 
+def seed(scores: dict[str, Any], names: Iterable[str]) -> None:
+    """Mark every expected scoring key as unscored (`None`) before invocation, so a
+    trace records which signals should have run even when scoring fails midway.
+    Overwrites: a re-scoring attempt resets its own names, so stale values from a
+    previous attempt never read as fresh."""
+    for name in names:
+        scores[name] = None
+
+
+def unseed(scores: dict[str, Any], name: str) -> None:
+    """Drop a still-unscored seed — a handler returning keyed scores records under
+    its result's keys, not its function name."""
+    if scores.get(name) is None:
+        scores.pop(name, None)
+
+
 def mark(attr: str, **extra: Any) -> Callable[[F], F]:
     def decorator(f: F) -> F:
         setattr(f, attr, True)

verifiers/v1/utils/platform.py

@@ -93,7 +93,8 @@ def trace_to_sample(
     # Flatten sub-rewards to top-level keys the way v0 does (raw scores, as v0's
     # per-function outputs were); env metrics stay nested.
     for name, reward in trace.rewards.items():
-        sample.setdefault(name, reward.score)
+        if reward is not None:
+            sample.setdefault(name, reward.score)
     return sample
 
 
@@ -128,8 +129,15 @@ def _run_metrics(episodes: list[Episode], traces: list[Trace]) -> dict[str, Any]
     sums: dict[str, float] = {}
     counts: dict[str, int] = {}
     for trace in scored:
-        scores = {name: reward.score for name, reward in trace.rewards.items()}
-        for name, value in {**scores, **trace.metrics}.items():
+        scores = {
+            name: reward.score
+            for name, reward in trace.rewards.items()
+            if reward is not None
+        }
+        metrics = {
+            name: value for name, value in trace.metrics.items() if value is not None
+        }
+        for name, value in {**scores, **metrics}.items():
             sums[name] = sums.get(name, 0.0) + value
             counts[name] = counts.get(name, 0) + 1
     n = len(scored)

{verifiers-0.2.2.dev73.dist-info → verifiers-0.2.2.dev74.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev73
+Version: 0.2.2.dev74
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev73.dist-info → verifiers-0.2.2.dev74.dist-info}/RECORD

@@ -173,15 +173,15 @@ verifiers/v1/env.py,sha256=g6fpoG-Z9lM3vTKrrU-QRJY3S88_y9zsPHwBvfs5vdY,17865
 verifiers/v1/episode.py,sha256=YtuRTV_PpwDglXFeN6UNg8IswpP1HseFLCYtascGMNI,3218
 verifiers/v1/errors.py,sha256=kQeEPX06TwAIuiz7MlWFYirrpKbx5STi6A5b_dtQQoA,6885
 verifiers/v1/graph.py,sha256=aBAzh1Ibp3klKNYjvzNyJSbjV8Oi5TJ791dN7o6ex1E,29070
-verifiers/v1/harness.py,sha256=i3pYHmdVPD7RtfZ-MAHlctaoLO8FxazCPmNy7mtPoSk,10954
+verifiers/v1/harness.py,sha256=eKuqvYbQjnCMavkiYWZQDHJgH08m6SJ4PuevPuShjS0,11097
 verifiers/v1/judge.py,sha256=Pvr0C41ah1qkNSZ0WXdkDvBMfv5YSfY1REHncwlJ8cg,9346
 verifiers/v1/legacy.py,sha256=eLsFiwihOMh4zS_u1V7DXF22r9beoG4k2JGwNEcSPfg,22872
 verifiers/v1/rollout.py,sha256=s1f1VZAAAI89j3y9ceq_E4mvX0F7cFQAnyx27SZsjvA,17881
 verifiers/v1/session.py,sha256=tNAraQfiJKpWqgBFNLyPhfHiw0JhzzkyOfRcKWGnR2Q,6939
 verifiers/v1/state.py,sha256=EckF2bWp-vV4b1jYJ9sLI5xrfGuI5spIgYYwW926toI,595
-verifiers/v1/task.py,sha256=D5PKxu6WlqCjv7hE311aaNvBA3QWVWDuPIEwDsq-J7g,9075
+verifiers/v1/task.py,sha256=jwMKiKlMtksTd8j2dcDlFtb6LC8BRE-N5jkNXlMp2jc,9457
 verifiers/v1/taskset.py,sha256=fp2E0IEhL_Ybj9cegZwljfTmW27p_30zTWHFKw_kXE4,4374
-verifiers/v1/trace.py,sha256=_bGHOrE6fRLGm0JYc0PrrrO3lyr-qqOF5Y1bHfViwR0,19347
+verifiers/v1/trace.py,sha256=Z4uK-lYAbl-6CHFhTo-gadHnbi63LC0sJtUERC_m5ss,19477
 verifiers/v1/types.py,sha256=1PxamJspmoTc3OlFZAwH6o_2i_Tg9ZNvHxttIHWIxq8,8948
 verifiers/v1/acp/__init__.py,sha256=9SYCFtzGUM_wH98ldpVLcFhoq2M999jbG0tU5ODY70U,2297
 verifiers/v1/acp/_runner.py,sha256=BcYaNZHuawzCgxOVdhiF6PY_B1HMxIA1MwHy0mOzhSk,7740
@@ -196,7 +196,7 @@ verifiers/v1/cli/serve.py,sha256=_mdJaHVVe4pXzcpAot0ugcmiN3ra2gS4GrGmLghnX5M,286
 verifiers/v1/cli/validate.py,sha256=7ax6FqIzNBSYjd3JXK4v7Vbq34Rozyh1OvGpYnGqlSg,10266
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
-verifiers/v1/cli/dashboard/eval.py,sha256=nQ24ptQ_6ylmrB9VoZkgga1CLp_7sXZueC_MC4O9Dow,33452
+verifiers/v1/cli/dashboard/eval.py,sha256=4j2YCPyzNHDT7d8MwqYR3C906lzwDVAd3FxgFYFgsw0,33588
 verifiers/v1/cli/dashboard/replay.py,sha256=CvRVaf0dUum5v0IzPQbFPPikBQmTrkMeaunGfVMTWHc,2755
 verifiers/v1/cli/dashboard/validate.py,sha256=rLsQ_31DjTIZpC_VO7zSG41ncUEBxqCx8mL9nTnGiMc,3650
 verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -234,7 +234,7 @@ verifiers/v1/dialects/chat.py,sha256=R_otqi5N0snB1QVKIYloNeLHR9kc7OmI06HYg1ir_Lw
 verifiers/v1/dialects/responses.py,sha256=ismnxUikPcPUIm80FUTPBbj-Cn1e0yGAsISMEseAWRc,13679
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=6vwtMCQ_cuvuubCF5-nrE5W7gGgzE-LIPKorkM5DXdw,309
-verifiers/v1/envs/agentic_judge/env.py,sha256=jreQPnbp8pJQSyZW0Me2IW-JipYsczqgohfM4-JCz9c,16968
+verifiers/v1/envs/agentic_judge/env.py,sha256=6QrfuIlg-9MkcqIn5_6F3qJ6ddLJNaeYsZRJC9EzGkw,17011
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/single_agent/__init__.py,sha256=r0edAc_6gtHBhPzeBGVcdHZZqSdCPZnLMje6eVgyz5U,138
@@ -324,7 +324,7 @@ verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuF
 verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
 verifiers/v1/utils/artifacts.py,sha256=vOzbPSuMirUecyylrhvLZ_vAcTd6JTJBztGPDQ0ANNk,6706
 verifiers/v1/utils/compile.py,sha256=Oh4hDVRNsql_BSOMh4kDEuze9_ZF17Q7uzBhhU70UZA,5663
-verifiers/v1/utils/decorators.py,sha256=9CcCwgZlRjVGPqT5G1CFJUHP4OIMuBGkUJZkcI6dnbk,3865
+verifiers/v1/utils/decorators.py,sha256=DG_K3HMiWqRHsY1bA7FJD3uTHUVuLw6SUJV-PnNRifA,4538
 verifiers/v1/utils/format.py,sha256=NfQo9M5KMzWCZpQaet5YtsJx56vCKAZPfbjZZJLJhhM,2187
 verifiers/v1/utils/generic.py,sha256=2VIN9RapMO9pVkFHB1UKUHck1vlTJ1pjfad2NzxuuXA,2055
 verifiers/v1/utils/git.py,sha256=MTRinOvG4EddfjdahHkFZUihHmhJ37n5ROtgjzjE1NA,8366
@@ -334,12 +334,12 @@ verifiers/v1/utils/interrupt.py,sha256=F-KKhc5ndPJJfhd3SuMqyqhXhA32FhCRy5KWFJpEo
 verifiers/v1/utils/loaders.py,sha256=NSIBdNU3JVeyddL-dM4dJt6khCHyjCgzJuFtLrgiwD0,9989
 verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y,2084
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
-verifiers/v1/utils/platform.py,sha256=MWZmvyj0iyzNIO0xRhlkCXtFF1plq6EJi8rWlvrFtvI,11363
+verifiers/v1/utils/platform.py,sha256=56Ixmk1SER6q5LvzyYcA0hgzGpzhxKZL8Hiqp4c_XVc,11576
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev73.dist-info/METADATA,sha256=DU0GJRGJc9sAtCc2vIZa5E-O-4ubaCtIXJiCZQHs2rc,4545
-verifiers-0.2.2.dev73.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev73.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev73.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev73.dist-info/RECORD,,
+verifiers-0.2.2.dev74.dist-info/METADATA,sha256=mYHTNMUxzf9tA02Dz787WOdGwU54MPGEJBgTSFG6aSs,4545
+verifiers-0.2.2.dev74.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev74.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev74.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev74.dist-info/RECORD,,