verifiers 0.1.15.dev391__tar.gz → 0.1.15.dev393__tar.gz

{verifiers-0.1.15.dev391 → verifiers-0.1.15.dev393}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev391
+Version: 0.1.15.dev393
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev391 → verifiers-0.1.15.dev393}/verifiers/v1/cli/dashboard/eval.py

@@ -1,9 +1,10 @@
 """The eval `--rich` dashboard: a config overview, a progress bar, and one line per rollout.
 
-Reads each `Rollout.trace`/`phase` every tick — no extra plumbing. Rows are colored by
-phase/outcome: setup (yellow ○), running (cyan ●), finalize (magenta ◑), scoring (blue ◐),
-success (green ✓), error (red ✗) — the reward shows only once a rollout is fully scored (phase
-DONE), so it never flips as scoring lands. A task's rollouts are grouped adjacently and joined
+Reads each `Rollout.trace`/`phase` every tick — no extra plumbing. Each row carries a bracketed
+phase marker that reads at a glance — `[setup]` (yellow), `[rollout]` (cyan), `[finalize]`
+(magenta), `[scoring]` (blue), `[success]` (green), `[error]` (red) — padded so the brackets
+line up in a column down the left edge. The reward shows only once a rollout is fully scored
+(phase DONE), so it never flips as scoring lands. A task's rollouts are grouped adjacently and joined
 by a left brace (╭│╰), so an episode (a task's n rollouts) reads as a unit. Every started
 rollout stays on screen (finished ones keep their result); the overview + progress sit on top,
 above a rule.
@@ -13,6 +14,7 @@ import contextlib
 import time
 
 from rich.console import Console, Group
+from rich.markup import escape
 from rich.progress_bar import ProgressBar
 from rich.rule import Rule
 from rich.table import Table
@@ -42,13 +44,20 @@ _STYLE = {
     "success": "green",
     "error": "red",
 }
+_MARK_LABEL = {
+    "setup": "setup",
+    "running": "rollout",
+    "finalize": "finalize",
+    "scoring": "scoring",
+    "success": "success",
+    "error": "error",
+}
+_MARK_WIDTH = max(len(label) for label in _MARK_LABEL.values())
+# Each label padded to a common width and bracketed, so the `[ ]` line up in a column down the
+# left edge (label left-aligned inside) — the current phase reads at a glance. `escape` keeps the
+# brackets literal: Rich parses `[label]` in a cell as markup and would otherwise drop it.
 _MARK = {
-    "setup": "○",
-    "running": "●",
-    "finalize": "◑",
-    "scoring": "◐",
-    "success": "✓",
-    "error": "✗",
+    state: escape(f"[{label:<{_MARK_WIDTH}}]") for state, label in _MARK_LABEL.items()
 }
 
 
@@ -185,10 +194,12 @@ def _breakdown(done: list[Trace]) -> Table | None:
 
 
 def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
-    """Input/output tokens for the main branch: output is every assistant (completion) token
-    generated across the branch's turns; input is the last turn's prompt — the full final
-    context the model saw. (Output can exceed the final context — reasoning tokens count toward
-    completions but aren't re-fed — so it's not derived by subtraction.)
+    """Input/output tokens summed across all branches: per branch, output is every assistant
+    (completion) token generated across its turns and input is its last turn's prompt — the full
+    final context the model saw. A rollout yields one training sample per branch (a linear trace
+    is a single branch; compaction and subagents add more), so the totals sum them — matching
+    `Trace.prompt_len` / `Trace.completion_len`. (Output can exceed the final context — reasoning
+    tokens count toward completions but aren't re-fed — so it's not derived by subtraction.)
 
     Prefers the token-id counts; falls back to provider-reported usage when the endpoint returns
     no token ids (e.g. plain OpenAI completions), so the counts aren't shown as 0/0. Returns
@@ -198,11 +209,8 @@ def _tokens(trace: Trace) -> tuple[int, int, int | None, int | None, int]:
     reasoning = usage.reasoning_tokens if usage else None
     branches = trace.branches
     nbranches = len(branches)
-    if not branches or not branches[0].nodes:
-        return 0, 0, cached, reasoning, nbranches
-    b = branches[0]
-    prompt = b.prompt_len or b.num_prompt_tokens
-    completion = b.completion_len or b.num_completion_tokens
+    prompt = sum(b.prompt_len or b.num_prompt_tokens for b in branches)
+    completion = sum(b.completion_len or b.num_completion_tokens for b in branches)
     return prompt, completion, cached, reasoning, nbranches
 
 

{verifiers-0.1.15.dev391 → verifiers-0.1.15.dev393}/verifiers/v1/cli/dashboard/validate.py

@@ -1,8 +1,10 @@
 """The validate `--rich` dashboard: a taskset overview, a progress bar, and one row per task.
 
 The model-free counterpart of the eval dashboard — no rollout phases, tokens, turns, or
-reward, just each task's validation outcome: pending ○ / running ● / valid ✓ / invalid ✗ /
-error ✗ / timeout ⏱. The runner advances a `TaskProgress` per task; this reads them each tick.
+reward, just each task's validation outcome, shown as a bracketed marker that reads at a
+glance — `[pending]` / `[running]` / `[valid]` / `[invalid]` / `[error]` / `[timeout]`,
+padded so the brackets line up in a column. The runner advances a `TaskProgress` per task;
+this reads them each tick.
 """
 
 import contextlib
@@ -10,6 +12,7 @@ import time
 from dataclasses import dataclass
 
 from rich.console import Group
+from rich.markup import escape
 from rich.progress_bar import ProgressBar
 from rich.rule import Rule
 from rich.table import Table
@@ -27,14 +30,11 @@ _STYLE = {
     "error": "red",
     "timeout": "red",
 }
-_MARK = {
-    "pending": "○",
-    "running": "●",
-    "valid": "✓",
-    "invalid": "✗",
-    "error": "✗",
-    "timeout": "⏱",
-}
+_MARK_WIDTH = max(len(state) for state in _STYLE)
+# Each state name padded to a common width and bracketed, so the `[ ]` line up in a column with
+# the name left-aligned inside — the outcome reads at a glance. `escape` keeps the brackets
+# literal: Rich parses `[name]` in a cell as markup and would otherwise drop it.
+_MARK = {state: escape(f"[{state:<{_MARK_WIDTH}}]") for state in _STYLE}
 _DONE = ("valid", "invalid", "error", "timeout")