verifiers 0.1.15.dev403__tar.gz → 0.1.15.dev405__tar.gz

{verifiers-0.1.15.dev403 → verifiers-0.1.15.dev405}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev403
+Version: 0.1.15.dev405
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev403 → verifiers-0.1.15.dev405}/verifiers/v1/cli/dashboard/eval.py

@@ -273,19 +273,21 @@ def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
     (completion) token generated across its turns and input is its last turn's prompt — the full
     final context the model saw. A rollout yields one training sample per branch (a linear trace
     is a single branch; compaction and subagents add more), so the totals sum them — matching
-    `Trace.prompt_len` / `Trace.completion_len`. (Output can exceed the final context — reasoning
-    tokens count toward completions but aren't re-fed — so it's not derived by subtraction.)
-
-    Prefers the token-id counts; falls back to provider-reported usage when the endpoint returns
-    no token ids (e.g. plain OpenAI completions), so the counts aren't shown as 0/0. Returns
-    the branch count from the same derived view so each dashboard tick materializes it once."""
+    `Trace.num_input_tokens` / `Trace.num_output_tokens`. (Output can exceed the final context —
+    reasoning tokens count toward completions but aren't re-fed — so it's not derived by
+    subtraction.)
+
+    Both counts read token ids when present and fall back to provider-reported usage when the
+    endpoint returns no token ids (e.g. plain OpenAI completions), so the counts aren't shown as
+    0/0. Returns the branch count from the same derived view so each dashboard tick materializes
+    it once."""
     usage = trace.usage
     cached = usage.cached_input_tokens if usage else None
     reasoning = usage.reasoning_tokens if usage else None
     branches = trace.branches
     nbranches = len(branches)
-    prompt = sum(b.input_len for b in branches)
-    completion = sum(b.output_len for b in branches)
+    prompt = sum(b.num_input_tokens for b in branches)
+    completion = sum(b.num_output_tokens for b in branches)
     return prompt, completion, cached, reasoning, nbranches
 
 

{verifiers-0.1.15.dev403 → verifiers-0.1.15.dev405}/verifiers/v1/env.py

@@ -106,13 +106,13 @@ class EnvConfig(BaseConfig):
     capping is a framework concern, never an harness or task field."""
     max_input_tokens: int | None = None
     """Max input (prompt) tokens per rollout (None = no limit). Caps the trace's
-    `prompt_len`; framework-enforced between turns."""
+    `num_input_tokens`; framework-enforced between turns."""
     max_output_tokens: int | None = None
     """Max output (completion) tokens per rollout (None = no limit). Caps the trace's
-    `completion_len`; framework-enforced between turns."""
+    `num_output_tokens`; framework-enforced between turns."""
     max_total_tokens: int | None = None
     """Max total (prompt + completion) tokens per rollout (None = no limit). Caps the
-    trace's `total_tokens`; framework-enforced between turns."""
+    trace's `num_total_tokens`; framework-enforced between turns."""
     multiplex: int = Field(32, ge=1)
     """Rollouts that share one interception server (and, behind a remote runtime, one
     tunnel). N concurrent rollouts use ~N/multiplex servers + tunnels instead of one each —

{verifiers-0.1.15.dev403 → verifiers-0.1.15.dev405}/verifiers/v1/interception/server.py

@@ -90,8 +90,8 @@ class RolloutLimits:
     """Per-rollout framework limits (None = no cap), checked before each turn is served.
     The first limit reached refuses the turn — halting any harness, the same mechanism as
     a @stop — and becomes the trace's stop condition. Each caps a trace computed property:
-    `max_turns` -> num_turns, `max_input_tokens` -> prompt_len, `max_output_tokens` ->
-    completion_len, `max_total_tokens` -> total_tokens. Token caps are soft by one turn:
+    `max_turns` -> num_turns, `max_input_tokens` -> num_input_tokens, `max_output_tokens` ->
+    num_output_tokens, `max_total_tokens` -> num_total_tokens. Token caps are soft by one turn:
     they're checked between turns, so the turn that crosses a cap still completes."""
 
     max_turns: int | None = None
@@ -105,17 +105,17 @@ class RolloutLimits:
             return "max_turns"
         if (
             self.max_input_tokens is not None
-            and trace.prompt_len >= self.max_input_tokens
+            and trace.num_input_tokens >= self.max_input_tokens
         ):
             return "max_input_tokens"
         if (
             self.max_output_tokens is not None
-            and trace.completion_len >= self.max_output_tokens
+            and trace.num_output_tokens >= self.max_output_tokens
         ):
             return "max_output_tokens"
         if (
             self.max_total_tokens is not None
-            and trace.total_tokens >= self.max_total_tokens
+            and trace.num_total_tokens >= self.max_total_tokens
         ):
             return "max_total_tokens"
         return None

{verifiers-0.1.15.dev403 → verifiers-0.1.15.dev405}/verifiers/v1/runtimes/base.py

@@ -192,8 +192,8 @@ class Runtime(ABC):
                         {},
                     )
                     command = (
-                        f"{_ENSURE_UV}; uv sync --script {shlex.quote(path)} -q "
-                        f"&& uv python find --script {shlex.quote(path)}"
+                        f"{_ENSURE_UV}; uv sync --script {shlex.quote(path)} -q --no-config "
+                        f"&& uv python find --script {shlex.quote(path)} --no-config"
                     )
                     result = await self.run(["sh", "-c", command], env or {})
                     if result.exit_code != 0:

{verifiers-0.1.15.dev403 → verifiers-0.1.15.dev405}/verifiers/v1/trace.py

@@ -165,60 +165,42 @@ class Branch(StrictBaseModel):
         return merged if merged.shape[0] == total else None
 
     @property
-    def completion_len(self) -> int:
-        """All assistant-generated (model-sampled) tokens across this branch."""
-        return sum(sum(n.mask) for n in self.nodes)
-
-    @property
-    def total_tokens(self) -> int:
+    def num_total_tokens(self) -> int:
         """This branch's full sequence length (final-turn prompt + every completion)."""
         return sum(len(n.token_ids) for n in self.nodes)
 
-    @property
-    def prompt_len(self) -> int:
-        """Input context size: the final-turn prompt = full sequence minus the last completion."""
-        last_completion = next(
-            (sum(n.mask) for n in reversed(self.nodes) if any(n.mask)), 0
-        )
-        return self.total_tokens - last_completion
-
     @property
     def usage(self) -> Usage | None:
         """Provider-reported usage summed over model calls in this branch."""
         return Usage.aggregate(n.usage for n in self.nodes if n.usage is not None)
 
     @property
-    def num_prompt_tokens(self) -> int:
-        """Final-turn input tokens from provider-reported usage — a fallback for display when
-        the endpoint returns no token ids (so `prompt_len` is 0); 0 if no usage was reported."""
+    def num_input_tokens(self) -> int:
+        """Input-context size: the final-turn prompt (full sequence minus the last completion).
+        Read from token ids when present (training), else from provider-reported usage so eval
+        clients that return no token ids don't report 0."""
+        last_completion = next(
+            (sum(n.mask) for n in reversed(self.nodes) if any(n.mask)), 0
+        )
+        token_len = self.num_total_tokens - last_completion
+        if token_len:
+            return token_len
         last = next(
             (n.usage for n in reversed(self.nodes) if n.usage is not None), None
         )
         return last.input_tokens if last else 0
 
     @property
-    def num_completion_tokens(self) -> int:
-        """All completion tokens across the branch from provider-reported usage — a fallback for
-        display when the endpoint returns no token ids; 0 if no usage was reported."""
+    def num_output_tokens(self) -> int:
+        """All assistant-generated (completion) tokens across this branch (reasoning included, so
+        it can exceed the final context). Read from token ids when present (training), else from
+        provider-reported usage so eval clients that return no token ids don't report 0."""
+        token_len = sum(sum(n.mask) for n in self.nodes)
+        if token_len:
+            return token_len
         usage = self.usage
         return usage.completion_tokens if usage else 0
 
-    @property
-    def input_len(self) -> int:
-        """Best-available input-context size: the token-id `prompt_len`, falling back to
-        provider-reported `num_prompt_tokens` when the endpoint returns no token ids (so it isn't
-        reported as 0). For display/metrics — use `prompt_len` when you need the strict token-id
-        count (e.g. token-cap enforcement)."""
-        return self.prompt_len or self.num_prompt_tokens
-
-    @property
-    def output_len(self) -> int:
-        """Best-available completion size: the token-id `completion_len`, falling back to
-        provider-reported `num_completion_tokens` when the endpoint returns no token ids (so it
-        isn't reported as 0). For display/metrics — use `completion_len` when you need the strict
-        token-id count (e.g. training, token-cap enforcement)."""
-        return self.completion_len or self.num_completion_tokens
-
 
 _NODE_DUMP_EXCLUDE: dict = {
     "nodes": {"__all__": {"multi_modal_data", "routed_experts"}}
@@ -264,7 +246,7 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
     extra_usage: list[Usage] = Field(default_factory=list)
     """Token usage from model calls that aren't part of the message graph — LLM judges and other
     auxiliary scoring calls (see `verifiers.v1.judge`). Kept separate from `usage` (the agent's own
-    spend) and off the graph, so branch/turn counts and the trainer's token math (`total_tokens`,
+    spend) and off the graph, so branch/turn counts and the trainer's token math (`num_total_tokens`,
     `branches`) see only sampled nodes; consumers that want a grand total add the two (e.g. the eval
     dashboard shows the agent's usage and `+judge` separately)."""
 
@@ -298,22 +280,22 @@ class Trace(StrictBaseModel, Generic[TaskT, StateT]):
         return next((n for n in reversed(self.nodes) if n.sampled), None)
 
     @property
-    def prompt_len(self) -> int:
+    def num_input_tokens(self) -> int:
         """Total input context summed over branches (each branch's final-turn prompt) —
         the trajectory yields one training sample per branch, so totals aggregate them."""
-        return sum(branch.prompt_len for branch in self.branches)
+        return sum(branch.num_input_tokens for branch in self.branches)
 
     @property
-    def completion_len(self) -> int:
+    def num_output_tokens(self) -> int:
         """Total assistant-generated (completion) tokens summed over branches — every token
         the model produced (reasoning included), so it can exceed the final context size."""
-        return sum(branch.completion_len for branch in self.branches)
+        return sum(branch.num_output_tokens for branch in self.branches)
 
     @property
-    def total_tokens(self) -> int:
+    def num_total_tokens(self) -> int:
         """Total sequence length summed over branches (each branch's final-turn prompt +
         completion) — used for token batching."""
-        return sum(branch.total_tokens for branch in self.branches)
+        return sum(branch.num_total_tokens for branch in self.branches)
 
     @property
     def usage(self) -> Usage | None: