verifiers 0.4.1.dev49__tar.gz → 0.4.1.dev50__tar.gz

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev49
+Version: 0.4.1.dev50
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev50}/verifiers/v1/cli/replay.py

@@ -1,7 +1,8 @@
 """Offline scoring for saved traces.
 
-Replay clears each trace's scores and recomputes everything computable from the saved
-transcript — trace-only handlers plus the layered config's judges. Runtime-requiring
+Replay re-scores each trace with `Task.rescore`: its scores cleared, everything
+computable from the saved transcript recomputed — trace-only handlers plus the layered
+config's judges. Runtime-requiring
 signals don't run offline, so a replay carries offline scores only; the source run keeps
 the runtime-recorded values. Its saved config is the base for replay-specific overrides.
 Multi-agent runs don't support replay: re-scoring is per trace, and the env's
@@ -152,16 +153,13 @@ async def run_replay(config: ReplayConfig, source: Path, out: Path) -> list[Trac
                 st.state, st.detail, st.end = "skipped", "rollout errored", time.time()
             else:
                 st.state = "running"
-                # Clear the recorded scores; only offline-recomputable ones come back.
-                trace.info.pop("judge_calls", None)
-                trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
                 # The declared Task for a rebuilt row, the base Task for the
                 # WireTaskData fallback.
                 task = (task_cls if isinstance(row, data_cls) else Task)(
                     row, config=config.taskset.task
                 )
                 try:
-                    await task.score(trace)
+                    await task.rescore(trace)
                     st.state, st.detail = "scored", f"reward {trace.reward:.3f}"
                 except Exception as exc:
                     st.state, st.detail = "error", type(exc).__name__

{verifiers-0.4.1.dev49 → verifiers-0.4.1.dev50}/verifiers/v1/task.py

@@ -224,6 +224,15 @@ class Task(Generic[DataT, StateT, ConfigT]):
         as a rollout would."""
         return None
 
+    async def rescore(self, trace: Trace) -> None:
+        """Score a finished trace again, offline: clear its rewards, metrics and judge
+        calls, then run this task's signals that need only the trace."""
+        if trace.has_error:
+            raise ValueError("the rollout errored: its transcript is incomplete")
+        trace.info.pop("judge_calls", None)
+        trace.rewards, trace.metrics, trace.extra_usage = {}, {}, []
+        await self.score(trace)
+
     def defer_scoring(self) -> Self:
         """An independent copy whose task signals are deferred.