verifiers 0.2.1.dev43__tar.gz → 0.2.1.dev44__tar.gz

{verifiers-0.2.1.dev43 → verifiers-0.2.1.dev44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev43
+Version: 0.2.1.dev44
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev43 → verifiers-0.2.1.dev44}/verifiers/v1/cli/dashboard/eval.py

@@ -363,17 +363,14 @@ def _breakdown(done: list[Trace]) -> Table | None:
 
 def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
     """Input/output tokens summed across all branches: per branch, output is every assistant
-    (completion) token generated across its turns and input is its last turn's prompt — the full
-    final context the model saw. A rollout yields one training sample per branch (a linear trace
-    is a single branch; compaction and subagents add more), so the totals sum them — matching
-    `Trace.num_input_tokens` / `Trace.num_output_tokens`. (Output can exceed the final context —
-    reasoning tokens count toward completions but aren't re-fed — so it's not derived by
-    subtraction.)
-
-    Both counts read token ids when present and fall back to provider-reported usage when the
-    endpoint returns no token ids (e.g. plain OpenAI completions), so the counts aren't shown as
-    0/0. Returns the branch count from the same derived view so each dashboard tick materializes
-    it once."""
+    (completion) token generated across its turns and input is the fed-in tokens counted once
+    (system + user + tool) — the final sequence minus everything the model generated. A rollout
+    yields one training sample per branch (a linear trace is a single branch; compaction and
+    subagents add more), so the totals sum them — matching `Trace.num_input_tokens` /
+    `Trace.num_output_tokens`, whose sum is `num_total_tokens`.
+
+    Both counts come from provider-reported usage. Returns the branch count from the same derived
+    view so each dashboard tick materializes it once."""
     usage = trace.usage
     cached = usage.cached_input_tokens if usage else None
     reasoning = usage.reasoning_tokens if usage else None

{verifiers-0.2.1.dev43 → verifiers-0.2.1.dev44}/verifiers/v1/trace.py

@@ -167,36 +167,37 @@ class Branch(StrictBaseModel):
         )
         return KeptTokens(ids=ids, counts=np.concatenate(counts_parts))
 
-    @property
-    def num_total_tokens(self) -> int:
-        return sum(len(n.token_ids) for n in self.nodes)
-
     @property
     def usage(self) -> Usage | None:
         return Usage.aggregate(n.usage for n in self.nodes if n.usage is not None)
 
     @property
-    def num_input_tokens(self) -> int:
-        """Final-turn prompt size, falling back to provider usage without token IDs."""
-        last_completion = next(
-            (sum(n.mask) for n in reversed(self.nodes) if any(n.mask)), 0
-        )
-        token_len = self.num_total_tokens - last_completion
-        if token_len:
-            return token_len
-        last = next(
+    def last_usage(self) -> Usage | None:
+        """Provider usage from the final model call on this branch — the full context it saw."""
+        return next(
             (n.usage for n in reversed(self.nodes) if n.usage is not None), None
         )
-        return last.input_tokens if last else 0
+
+    @property
+    def num_total_tokens(self) -> int:
+        """Final sequence length: the last call's prompt + completion. Earlier turns' context
+        is already contained in that prompt, so re-sent tokens are counted once rather than
+        summed per turn."""
+        last = self.last_usage
+        return last.total_tokens if last is not None else 0
 
     @property
     def num_output_tokens(self) -> int:
-        """Sampled tokens, falling back to provider usage without token IDs."""
-        token_len = sum(sum(n.mask) for n in self.nodes)
-        if token_len:
-            return token_len
+        """Every model-generated token across all turns (completions, reasoning included)."""
         usage = self.usage
-        return usage.completion_tokens if usage else 0
+        return usage.completion_tokens if usage is not None else 0
+
+    @property
+    def num_input_tokens(self) -> int:
+        """Tokens fed to the model, counted once: the final sequence minus everything the model
+        generated (i.e. system + user + tool inputs). Not the last prompt — re-sent context is
+        not double-counted."""
+        return self.num_total_tokens - self.num_output_tokens
 
 
 _NODE_DUMP_EXCLUDE: dict = {
@@ -282,17 +283,17 @@ class Trace(StrictBaseModel, Generic[DataT, StateT]):
 
     @property
     def num_input_tokens(self) -> int:
-        """Final-turn prompt sizes summed across training branches."""
+        """Fed-in tokens (system + user + tool), counted once, summed across branches."""
         return sum(branch.num_input_tokens for branch in self.branches)
 
     @property
     def num_output_tokens(self) -> int:
-        """Model-sampled tokens summed across training branches."""
+        """Model-generated tokens across all turns, summed across branches."""
         return sum(branch.num_output_tokens for branch in self.branches)
 
     @property
     def num_total_tokens(self) -> int:
-        """Sequence lengths summed across training branches for token batching."""
+        """Final sequence lengths (last prompt + completion) summed across branches."""
         return sum(branch.num_total_tokens for branch in self.branches)
 
     @property