verifiers 0.2.1.dev59__tar.gz → 0.2.1.dev60__tar.gz

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev59
+Version: 0.2.1.dev60
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev60}/verifiers/v1/__init__.py

@@ -105,9 +105,11 @@ from verifiers.v1.trace import (
     Branch,
     Error,
     EvalRunInfo,
+    GenerationSpan,
     ModelCall,
     RunInfo,
     TimeSpan,
+    TimeSplit,
     Timing,
     Trace,
     TraceTask,
@@ -184,6 +186,8 @@ __all__ = [
     "KeptTokens",
     "Timing",
     "TimeSpan",
+    "TimeSplit",
+    "GenerationSpan",
     "Error",
     # decorators
     "stop",

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev60}/verifiers/v1/cli/dashboard/eval.py

@@ -300,6 +300,7 @@ def _breakdown(done: list[Trace]) -> Table | None:
     have_cost = have_cached = have_reasoning = have_judge = False
     phase_secs: dict[str, float] = {}
     phase_count: dict[str, int] = {}
+    model_secs = harness_secs = 0.0
     for trace in done:
         prompt, completion, cached, reasoning, _ = _tokens(trace)
         total_in += prompt
@@ -326,6 +327,8 @@ def _breakdown(done: list[Trace]) -> Table | None:
             if span.end:  # phase was timed for this rollout
                 phase_secs[phase] = phase_secs.get(phase, 0.0) + span.duration
                 phase_count[phase] = phase_count.get(phase, 0) + 1
+        model_secs += trace.timing.generation.model.duration
+        harness_secs += trace.timing.generation.harness.duration
     if (
         total_in
         or total_out
@@ -353,11 +356,18 @@ def _breakdown(done: list[Trace]) -> Table | None:
                 cost += f" + {format_cost_usd(total_judge_cost)} judge"
             usage.append(cost)
         grid.add_row("usage", "  ·  ".join(usage))
-    time_segments = [
-        f"{phase} {format_time(phase_secs[phase] / phase_count[phase])}"
-        for phase in ("boot", "setup", "generation", "finalize", "scoring")
-        if phase_count.get(phase)
-    ]
+    time_segments = []
+    for phase in ("boot", "setup", "generation", "finalize", "scoring"):
+        count = phase_count.get(phase)
+        if not count:
+            continue
+        segment = f"{phase} {format_time(phase_secs[phase] / count)}"
+        if phase == "generation":
+            segment += (
+                f" (model {format_time(model_secs / count)}"
+                f" + harness {format_time(harness_secs / count)})"
+            )
+        time_segments.append(segment)
     if time_segments:
         grid.add_row("time", "  ·  ".join(time_segments))
     return grid if grid.row_count else None

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev60}/verifiers/v1/cli/debug.py

@@ -243,6 +243,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
     except Exception as e:
         record_debug_error(trace, debug, e, setup_timeout, config.timeout.total)
     finally:
+        trace.split_generation()
         trace.info["debug"] = debug
         try:
             await runtime.stop()

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev60}/verifiers/v1/legacy.py

@@ -32,7 +32,16 @@ from verifiers.v1.serve.types import (
 )
 from verifiers.v1.task import WireTaskData
 from verifiers.v1 import graph
-from verifiers.v1.trace import Error, ModelCall, TimeSpan, Timing, Trace, TraceTask
+from verifiers.v1.trace import (
+    Error,
+    GenerationSpan,
+    ModelCall,
+    TimeSpan,
+    TimeSplit,
+    Timing,
+    Trace,
+    TraceTask,
+)
 from verifiers.v1.types import (
     AssistantMessage,
     Response,
@@ -185,7 +194,8 @@ def _to_v1_tokens(raw: Any) -> TurnTokens | None:
 
 def _timing(raw: Any) -> Timing:
     """Map the v0 timing record's generation/scoring durations onto a v1 ``Timing``
-    (we only have durations, so each span is encoded as start=0, end=duration)."""
+    (we only have durations, so each span is encoded as start=0, end=duration).
+    v0's per-turn ``model``/``env`` span collections carry the generation split."""
 
     def _dur(node: Any) -> float:
         if isinstance(node, dict):
@@ -198,7 +208,12 @@ def _timing(raw: Any) -> Timing:
 
     raw = raw or {}
     return Timing(
-        generation=TimeSpan(start=0.0, end=_dur(raw.get("generation"))),
+        generation=GenerationSpan(
+            start=0.0,
+            end=_dur(raw.get("generation")),
+            model=TimeSplit(duration=_dur(raw.get("model"))),
+            harness=TimeSplit(duration=_dur(raw.get("env"))),
+        ),
         scoring=TimeSpan(start=0.0, end=_dur(raw.get("scoring"))),
     )
 

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev60}/verifiers/v1/rollout.py

@@ -249,6 +249,7 @@ class Rollout:
             ):
                 if span.start and not span.end:
                     span.end = now
+            trace.split_generation()
             try:
                 await runtime.stop()
             except Exception:

{verifiers-0.2.1.dev59 → verifiers-0.2.1.dev60}/verifiers/v1/trace.py

@@ -49,11 +49,30 @@ class TimeSpan(StrictBaseModel):
         return max(0.0, self.end - self.start) if self.end else 0.0
 
 
+class TimeSplit(StrictBaseModel):
+    """A span's share attributed to one side of a split: disjoint sub-intervals summed
+    into a duration, so there is no single start/end. Serialized, unlike a span's
+    derived `duration`, since it cannot be recomputed from two timestamps."""
+
+    duration: float = 0.0
+
+
+class GenerationSpan(TimeSpan):
+    """The generation span plus its split into time inside model calls (`model`) vs.
+    outside them (`harness`: harness logic, tools, user simulation). Stamped from the
+    recorded model calls' spans by `Trace.split_generation` when the span closes, with
+    `model.duration + harness.duration == duration` — concurrent calls (subagent forks)
+    are clamped to the span, saturating the model share."""
+
+    model: TimeSplit = Field(default_factory=TimeSplit)
+    harness: TimeSplit = Field(default_factory=TimeSplit)
+
+
 class Timing(StrictBaseModel):
     start: float = Field(default_factory=time.time)
     boot: TimeSpan = Field(default_factory=TimeSpan)
     setup: TimeSpan = Field(default_factory=TimeSpan)
-    generation: TimeSpan = Field(default_factory=TimeSpan)
+    generation: GenerationSpan = Field(default_factory=GenerationSpan)
     finalize: TimeSpan = Field(default_factory=TimeSpan)
     scoring: TimeSpan = Field(default_factory=TimeSpan)
 
@@ -537,6 +556,18 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
         if self.stop_condition is None:
             self.stop_condition = condition
 
+    def split_generation(self) -> None:
+        """Stamp the closed generation span's model/harness split: model time is the
+        sum of the recorded calls' spans (clamped to the span), harness the complement.
+        Every path that closes the span calls this — a span without model calls (e.g.
+        a debug action) is all harness time."""
+        gen = self.timing.generation
+        if not gen.end:
+            return
+        model = sum(call.time.duration for call in self.calls)
+        gen.model.duration = min(model, gen.duration)
+        gen.harness.duration = gen.duration - gen.model.duration
+
     def capture_error(self, error: Exception) -> None:
         self.errors.append(
             Error(