verifiers 0.2.1.dev19__tar.gz → 0.2.1.dev20__tar.gz

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev19
+Version: 0.2.1.dev20
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev20}/verifiers/v1/cli/dashboard/eval.py

@@ -40,6 +40,7 @@ _LABEL_WIDTH = len("timeouts")
 
 _STYLE = {
     "pending": "dim",
+    "boot": "orange3",
     "setup": "yellow",
     "running": "cyan",
     "finalize": "magenta",
@@ -49,6 +50,7 @@ _STYLE = {
 }
 _MARK_LABEL = {
     "pending": "pending",
+    "boot": "boot",
     "setup": "setup",
     "running": "rollout",
     "finalize": "finalize",
@@ -317,7 +319,7 @@ def _breakdown(done: list[Trace]) -> Table | None:
             if judge.cost is not None:
                 total_judge_cost += judge.cost
             have_judge = True
-        for phase in ("setup", "generation", "finalize", "scoring"):
+        for phase in ("boot", "setup", "generation", "finalize", "scoring"):
             span = getattr(trace.timing, phase)
             if span.end:  # phase was timed for this rollout
                 phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
@@ -351,7 +353,7 @@ def _breakdown(done: list[Trace]) -> Table | None:
         grid.add_row("usage", "  ·  ".join(usage))
     time_segments = [
         f"{phase} {format_time(phase_secs[phase] / phase_count[phase])}"
-        for phase in ("setup", "generation", "finalize", "scoring")
+        for phase in ("boot", "setup", "generation", "finalize", "scoring")
         if phase_count.get(phase)
     ]
     if time_segments:
@@ -383,11 +385,12 @@ def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
 
 
 def _started(rollout: Rollout) -> float:
-    # Sort key: when a rollout began (its setup start). A still-pending rollout has no trace
-    # yet, so it sorts last (+inf) — behind everything already in flight, in task order.
-    return (
-        rollout.trace.timing.setup.start if rollout.trace is not None else float("inf")
-    )
+    # Sort key: when a rollout began (its boot start; setup for pre-boot-span traces on
+    # resume). A still-pending rollout has no trace yet, so it sorts last (+inf) — behind
+    # everything already in flight, in task order.
+    if rollout.trace is None:
+        return float("inf")
+    return rollout.trace.timing.boot.start or rollout.trace.timing.setup.start
 
 
 def _groups(rollouts: list[Rollout]) -> list[list[Rollout]]:
@@ -450,14 +453,14 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
             )
             runtime = f"{runtime_type}({descriptor})" if descriptor else runtime_type
             turns = t.num_turns
-            start = t.timing.setup.start
+            start = t.timing.boot.start or t.timing.setup.start
             end = (
                 t.timing.scoring.end
                 or t.timing.finalize.end
                 or t.timing.generation.end
-                # a rollout that errored in setup has only setup.end — freeze there once done,
-                # else (still running) the timer would grow off `now` forever
-                or (t.timing.setup.end if t.is_completed else 0)
+                # a rollout that errored in boot/setup has only that span's end — freeze there
+                # once done, else (still running) the timer would grow off `now` forever
+                or ((t.timing.setup.end or t.timing.boot.end) if t.is_completed else 0)
                 or now
             )
             prompt, completion, cached, reasoning, nbranches = _tokens(t)

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev20}/verifiers/v1/cli/debug.py

@@ -138,15 +138,19 @@ def record_debug_error(
     setup_timeout: float | None,
     action_timeout: float | None,
 ) -> None:
-    if not trace.timing.setup.end:
-        trace.timing.setup.end = time.time()
-    if trace.timing.generation.start and not trace.timing.generation.end:
-        trace.timing.generation.end = time.time()
+    now = time.time()
+    for span in (trace.timing.boot, trace.timing.setup, trace.timing.generation):
+        if span.start and not span.end:
+            span.end = now
     in_action = bool(trace.timing.generation.start)
-    stage = "debug action" if in_action else "setup"
+    stage = (
+        "debug action" if in_action else "setup" if trace.timing.setup.start else "boot"
+    )
     timeout = action_timeout if in_action else setup_timeout
     error_start = (
-        trace.timing.generation.start if in_action else trace.timing.setup.start
+        trace.timing.generation.start
+        if in_action
+        else trace.timing.setup.start or trace.timing.boot.start
     )
     debug.update(error_info(error, error_start, timeout, stage))
     debug.setdefault("runtime", runtime_info(runtime))
@@ -226,8 +230,11 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         else task.data.timeout.setup
     )
     try:
-        trace.timing.setup.start = time.time()
+        trace.timing.boot.start = time.time()
         await runtime.start()
+        now = time.time()
+        trace.timing.boot.end = now
+        trace.timing.setup.start = now
         debug["runtime"] = runtime_info(runtime)
         await asyncio.wait_for(
             invoke(task.setup, {"trace": trace, "runtime": runtime}),

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev20}/verifiers/v1/rollout.py

@@ -37,6 +37,7 @@ logger = logging.getLogger(__name__)
 
 class Phase(StrEnum):
     PENDING = "pending"
+    BOOT = "boot"
     SETUP = "setup"
     RUNNING = "running"
     FINALIZE = "finalize"
@@ -108,8 +109,8 @@ class Rollout:
             state=state_cls(type(self.task))(),
         )
         self.trace = trace  # expose for the --rich dashboard
-        self.phase = Phase.SETUP  # leaving the queue: provisioning starts now
-        trace.timing.setup.start = time.time()
+        self.phase = Phase.BOOT  # leaving the queue: the runtime boots now
+        trace.timing.boot.start = time.time()
         self.runtime = make_runtime(self.runtime_config, name=trace.id)
         runtime = self.runtime
         trace.runtime = runtime.info
@@ -125,6 +126,10 @@ class Rollout:
         try:
             session = RolloutSession(ctx, trace, stops, self.limits)
             await runtime.start()
+            now = time.time()
+            trace.timing.boot.end = now
+            trace.timing.setup.start = now
+            self.phase = Phase.SETUP
             # Task setup and harness provisioning share one setup-stage deadline.
             setup_deadline = (
                 None
@@ -228,6 +233,7 @@ class Rollout:
             trace.is_completed = True
             now = time.time()
             for span in (
+                trace.timing.boot,
                 trace.timing.setup,
                 trace.timing.generation,
                 trace.timing.finalize,

{verifiers-0.2.1.dev19 → verifiers-0.2.1.dev20}/verifiers/v1/trace.py

@@ -47,6 +47,7 @@ class TimeSpan(StrictBaseModel):
 
 class Timing(StrictBaseModel):
     start: float = Field(default_factory=time.time)
+    boot: TimeSpan = Field(default_factory=TimeSpan)
     setup: TimeSpan = Field(default_factory=TimeSpan)
     generation: TimeSpan = Field(default_factory=TimeSpan)
     finalize: TimeSpan = Field(default_factory=TimeSpan)