verifiers 0.3.2.dev62__py3-none-any.whl → 0.3.2.dev64__py3-none-any.whl

verifiers/v1/cli/dashboard/eval.py

@@ -295,9 +295,22 @@ def _push_footer(push: "PushState | None") -> Group | None:
     return Group(Rule(style="dim"), line)
 
 
+class TraceStats:
+    """Resource totals for a dashboard frame, retained once the episode is final."""
+
+    def __init__(self, trace: Trace) -> None:
+        self.num_input_tokens = trace.num_input_tokens
+        self.num_output_tokens = trace.num_output_tokens
+        self.num_turns = trace.num_turns
+        self.num_branches = trace.num_branches
+        self.usage = trace.usage
+        self.judge_usage = Usage.aggregate(trace.extra_usage)
+
+
 def Progress(
     slots: list[RunSlot],
     start: float,
+    completed: dict[str, TraceStats],
     page: tuple[int, int] | None = None,
 ) -> Group:
     # On resume, `slots` includes the previous session's kept rollouts (as finished slots), so
@@ -332,7 +345,7 @@ def Progress(
         ProgressBar(total=total or 1, completed=len(done)),
         Text(stats),
     )
-    breakdown = _breakdown(scored, done_traces)
+    breakdown = _breakdown(scored, done_traces, completed)
     return Group(row, breakdown) if breakdown is not None else Group(row)
 
 
@@ -362,7 +375,9 @@ def _score(trace: Trace, source: str, name: str) -> float:
     return value if value is not None else 0.0
 
 
-def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
+def _breakdown(
+    scored: list[Trace], done: list[Trace], completed: dict[str, TraceStats]
+) -> Table | None:
     """Score rows read the policy view (`scored` — trainable traces); with several
     roles in play they split per role, each role averaging over its OWN traces (no
     dilution, so the split covers every role — an untrainable seat's received
@@ -401,9 +416,10 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
     phase_count: dict[str, int] = {}
     model_secs = harness_secs = 0.0
     for trace in done:
-        total_in += trace.num_input_tokens
-        total_out += trace.num_output_tokens
-        usage = trace.usage
+        stats = completed[trace.id]
+        total_in += stats.num_input_tokens
+        total_out += stats.num_output_tokens
+        usage = stats.usage
         if usage is not None:
             if usage.cached_input_tokens is not None:
                 total_cached += usage.cached_input_tokens
@@ -415,7 +431,7 @@ def _breakdown(scored: list[Trace], done: list[Trace]) -> Table | None:
                 total_cost += usage.cost
                 have_cost = True
         # Judge / auxiliary scoring calls (off the message graph) shown separately from the agent's.
-        judge = Usage.aggregate(trace.extra_usage)
+        judge = stats.judge_usage
         if judge is not None:
             total_judge_in += judge.input_tokens
             total_judge_out += judge.completion_tokens
@@ -530,7 +546,12 @@ def _brace(i: int, size: int) -> str:
     return "╭" if i == 0 else "╰" if i == size - 1 else "│"
 
 
-def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
+def Rows(
+    groups: list[list[RunSlot]],
+    now: float,
+    runtime_type: str,
+    completed: dict[str, TraceStats],
+) -> Table:
     # (brace, state, left sections, result, time); a slot contributes one row per live
     # trace (a multi-agent episode shows each role's trace), braced per task.
     rows: list[tuple[str, str, list[str], str, str]] = []
@@ -596,7 +617,8 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     runtime = f"{rt.type}({rt.id})" if rt.id else rt.type
                 else:
                     runtime = runtime_type
-                turns = t.num_turns
+                stats = completed[t.id] if slot.done else TraceStats(t)
+                turns = stats.num_turns
                 start = t.timing.boot.start or t.timing.setup.start
                 end = (
                     t.timing.scoring.end
@@ -611,9 +633,9 @@ def Rows(groups: list[list[RunSlot]], now: float, runtime_type: str) -> Table:
                     )
                     or now
                 )
-                prompt, completion = t.num_input_tokens, t.num_output_tokens
-                nbranches = t.num_branches
-                usage = t.usage
+                prompt, completion = stats.num_input_tokens, stats.num_output_tokens
+                nbranches = stats.num_branches
+                usage = stats.usage
                 cached = usage.cached_input_tokens if usage else None
                 reasoning = usage.reasoning_tokens if usage else None
                 cost = usage.cost if usage else None
@@ -786,16 +808,26 @@ def _render(
     pager: Pager,
     header: Group | Table,
     runtime_type: str,
+    completed: dict[str, TraceStats],
     push: "PushState | None" = None,
     tail: LogTail | None = None,
 ) -> Group:
     now = time.time()
+    # A trace can finish before other agents and env scoring update the episode.
+    # Only done slots are final; resumed episodes enter the view already done.
+    completed.update(
+        (trace.id, TraceStats(trace))
+        for slot in slots
+        if slot.done
+        for trace in slot.traces
+        if trace.id not in completed
+    )
     # The --push status line (and, on Ctrl-C, the cleanup notice) appear under the rollouts. Measure
     # the fixed top (header + progress + rule) and the footer so the rollout rows fill what's left;
     # page through them (timer / arrows) when they'd overflow (else rich truncates).
     footers = [f for f in (_push_footer(push), _interrupt_footer()) if f is not None]
     footer = Group(*footers) if footers else None
-    progress = Progress(slots, start)
+    progress = Progress(slots, start, completed)
     top = Group(header, progress, Rule(style="dim"))
     reserved = len(_CONSOLE.render_lines(top))
     if footer is not None:
@@ -813,12 +845,12 @@ def _render(
         return Group(*parts)
     page_groups, index, count = _paginate(_groups(slots), rows_per_page, pager, now)
     if count > 1:
-        progress = Progress(slots, start, page=(index + 1, count))
+        progress = Progress(slots, start, completed, page=(index + 1, count))
     parts = [
         header,
         progress,
         Rule(style="dim"),
-        Rows(page_groups, now, runtime_type),
+        Rows(page_groups, now, runtime_type, completed),
     ]
     if footer is not None:
         parts.append(footer)
@@ -833,6 +865,7 @@ async def dashboard(
     push: "PushState | None" = None,
 ):
     pager = Pager()
+    completed: dict[str, TraceStats] = {}
     warning = _warning(config)
     header = Group(warning, Text(""), Overview(config)) if warning else Overview(config)
     runtime_type = (
@@ -850,7 +883,9 @@ async def dashboard(
         else None
     )
     async with live_view(
-        lambda: _render(slots, start, pager, header, runtime_type, push, tail),
+        lambda: _render(
+            slots, start, pager, header, runtime_type, completed, push, tail
+        ),
         on_key=None if tail is not None else pager.on_key,
     ):
         yield

verifiers/v1/utils/artifacts.py

@@ -91,12 +91,33 @@ async def collect(
             raise RuntimeError(f"artifact {artifact.source!r} declared more than once")
         seen.add(artifact.source)
 
+    # Batch roots to save remote round trips. Leave room below the shell argument
+    # limit for the command itself and further quoting by runtime transports.
+    batches: list[list[str]] = [[]]
+    batch_bytes = 0
+    for artifact in entries:
+        source_bytes = len(shlex.quote(artifact.source).encode()) + 1
+        if batches[-1] and batch_bytes + source_bytes > 8 * 1024:
+            batches.append([])
+            batch_bytes = 0
+        batches[-1].append(artifact.source)
+        batch_bytes += source_bytes
+
+    existence: list[str] = []
+    for sources in batches:
+        output = await _run(
+            runtime,
+            f"for source in {shlex.join(sources)}; do "
+            'if test -e "$source" || test -L "$source"; then echo 1; else echo 0; fi; '
+            "done",
+            "check artifact roots",
+        )
+        existence.extend(output.splitlines())
     collected: dict[str, bytes | None] = {}
     budget = MAX_ARTIFACT_BYTES
-    for artifact in entries:
+    for artifact, exists in zip(entries, existence, strict=True):
         source = artifact.source
-        exists = f"test -e {shlex.quote(source)} || test -L {shlex.quote(source)}"
-        if (await runtime.run(["sh", "-c", exists], {})).exit_code != 0:
+        if exists != "1":
             if not artifact.required:
                 collected[source] = None
                 continue
@@ -205,8 +226,9 @@ def _validate_restore(root: str, archive: bytes | None) -> None:
         raise RuntimeError(f"unreadable artifact archive for {root!r}: {exc}") from exc
 
 
-async def _run(runtime: Runtime, command: str, action: str) -> None:
+async def _run(runtime: Runtime, command: str, action: str) -> str:
     result = await runtime.run(["sh", "-c", command], {})
     if result.exit_code:
         detail = (result.stderr or result.stdout).strip()[-500:]
         raise RuntimeError(f"failed to {action}: {detail}")
+    return result.stdout

{verifiers-0.3.2.dev62.dist-info → verifiers-0.3.2.dev64.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev62
+Version: 0.3.2.dev64
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev62.dist-info → verifiers-0.3.2.dev64.dist-info}/RECORD

@@ -29,7 +29,7 @@ verifiers/v1/cli/resume.py,sha256=hqV5AxxqeIInJeEZQerQznuijhMNijFEFcHZIltfLdE,47
 verifiers/v1/cli/validate.py,sha256=G3YXZTwxVDQIyR_ASapJceixOxRSJl69NnFPBqkF3N0,17173
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
-verifiers/v1/cli/dashboard/eval.py,sha256=IgyEVBfTcyuKw_rXaHrRRFX4ESlwsekaPi-bxDDdDKY,36633
+verifiers/v1/cli/dashboard/eval.py,sha256=hr2r54wvcMYw-CEjDsVxrBQgZGp9cGrgrsRtjaaSNPs,37822
 verifiers/v1/cli/dashboard/replay.py,sha256=_X5up9MJsRbd_sWe4LgNTPNT8k1ip4xNpo50hxI3Ljk,2697
 verifiers/v1/cli/dashboard/validate.py,sha256=xrpK3Y90JsoDCQYLLvvsnJ4CPRMvSSJIghfHK62E6FQ,3687
 verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -172,7 +172,7 @@ verifiers/v1/tasksets/textarena/__init__.py,sha256=Os2OlBY_pSH0B1DP32RitumgSLpq2
 verifiers/v1/tasksets/textarena/taskset.py,sha256=9fa_unlFiFZyuN4I02rW3x4qyRnFWXqwTUZTloQefu0,4433
 verifiers/v1/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/utils/aio.py,sha256=0yNFOqB2oO6Qktc6NI_ZSAGuo1aHsIK_wuT1BtmoZ3I,1532
-verifiers/v1/utils/artifacts.py,sha256=GowZubXGk6NhVgaAUKCVXYoBh_3L1Aw0LAHe6tNwl8g,8447
+verifiers/v1/utils/artifacts.py,sha256=PFrSkwAyvzHyT6JjlFeGOPIXDmS-uRajaN3am4HZ0TI,9263
 verifiers/v1/utils/compile.py,sha256=5pxy31wx2Q-3pQjOO5B9SH6OT-wwb8r4F1LAl6HMeQA,5835
 verifiers/v1/utils/decorators.py,sha256=vhoN6YoKb15-Fkqmc9RaNlUIbt9WRvNK4zAn2bAxfS4,5367
 verifiers/v1/utils/format.py,sha256=NfQo9M5KMzWCZpQaet5YtsJx56vCKAZPfbjZZJLJhhM,2187
@@ -189,8 +189,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev62.dist-info/METADATA,sha256=5hEsJTz4nby3P0DDkENjpoYc2Y2T5ikuW1S7W_lzZ1c,4204
-verifiers-0.3.2.dev62.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev62.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
-verifiers-0.3.2.dev62.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev62.dist-info/RECORD,,
+verifiers-0.3.2.dev64.dist-info/METADATA,sha256=uYX4LJNV3yeutj3Pwcw8IU3Fbz-CYNSzBwOs5vX50nY,4204
+verifiers-0.3.2.dev64.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev64.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
+verifiers-0.3.2.dev64.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev64.dist-info/RECORD,,