verifiers 0.1.15.dev152__tar.gz → 0.1.15.dev154__tar.gz

{verifiers-0.1.15.dev152 → verifiers-0.1.15.dev154}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev152
+Version: 0.1.15.dev154
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev152 → verifiers-0.1.15.dev154}/verifiers/scripts/eval.py

@@ -594,8 +594,8 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "--max-retries",
         type=int,
-        default=0,
-        help="Max retries for transient infrastructure errors (default: 0)",
+        default=3,
+        help="Max retries for transient infrastructure errors (default: 3)",
     )
     parser.add_argument(
         "--disable-env-server",
@@ -958,7 +958,7 @@ def main(argv: list[str] | None = None):
             num_examples=num_examples,
             rollouts_per_example=rollouts_per_example,
             max_concurrent=raw.get("max_concurrent", DEFAULT_MAX_CONCURRENT),
-            max_retries=raw.get("max_retries", 0),
+            max_retries=raw.get("max_retries", 3),
             num_workers=raw.get("num_workers", "auto"),
             disable_env_server=raw.get("disable_env_server", False),
             verbose=raw.get("verbose", False),

{verifiers-0.1.15.dev152 → verifiers-0.1.15.dev154}/verifiers/types.py

@@ -1329,7 +1329,7 @@ class EvalConfig(BaseModel):
     num_workers: int | str = "auto"
     independent_scoring: bool = False
     extra_env_kwargs: dict = {}
-    max_retries: int = 0
+    max_retries: int = 3
     disable_env_server: bool = False
     # logging
     verbose: bool = False

{verifiers-0.1.15.dev152 → verifiers-0.1.15.dev154}/verifiers/utils/display_utils.py

@@ -533,6 +533,25 @@ def _timing_parts(
     return parts
 
 
+def format_timing_plain(
+    setup: float = 0.0,
+    generation: float = 0.0,
+    scoring: float = 0.0,
+    overhead: float = 0.0,
+    model: float = 0.0,
+    env: float = 0.0,
+) -> str:
+    """Return a plain-text compact timing breakdown (no Rich styling)."""
+    parts = _timing_parts(setup, generation, scoring, overhead, model, env)
+    segments: list[str] = []
+    for label, value, subs in parts:
+        segment = f"{label} {value}"
+        if subs:
+            segment += " (" + " + ".join(f"{sl} {sv}" for sl, sv in subs) + ")"
+        segments.append(segment)
+    return " + ".join(segments)
+
+
 def format_timing_rich(
     setup: float = 0.0,
     generation: float = 0.0,

{verifiers-0.1.15.dev152 → verifiers-0.1.15.dev154}/verifiers/v1/harness.py

@@ -71,6 +71,7 @@ from .utils.sandbox_program_utils import (
     python_program_sandbox,
     run_sandbox_python_program,
 )
+from .utils.logging_utils import log_rollout_finish, log_rollout_start
 from .utils.prompt_utils import (
     SystemPrompt,
     SystemPromptStrategy,
@@ -241,6 +242,7 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
 
     async def run(self, task: Task, state: State | None = None) -> State:
         state = await self.init_state(task) if state is None else state
+        log_rollout_start(state)
         timing_recorded = False
         completed = False
         try:
@@ -272,6 +274,7 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
                     state.strip_runtime_handles()
             elif completed:
                 state.assert_serializable()
+            log_rollout_finish(state)
         return state
 
     def record_error(self, state: State, error: Error) -> None:

verifiers-0.1.15.dev154/verifiers/v1/utils/logging_utils.py

@@ -0,0 +1,57 @@
+import logging
+from collections import Counter
+
+from verifiers.utils.display_utils import format_numeric, format_timing_plain
+from verifiers.utils.logging_utils import truncate
+
+from ..state import State
+
+logger = logging.getLogger("verifiers.v1.rollout")
+
+
+def log_rollout_start(state: State) -> None:
+    logger.info(
+        f"Started example_id={state.get('example_id')} "
+        f"| trajectory_id={state.get('trajectory_id')}"
+    )
+
+
+def log_rollout_finish(state: State) -> None:
+    tools = Counter(
+        call["name"]
+        for step in state.get("trajectory") or []
+        for message in step.get("completion") or []
+        for call in message.get("tool_calls") or []
+    )
+    timing = state.get("timing") or {}
+    metrics = state.get("metrics") or {}
+
+    def duration(phase: str) -> float:
+        return (timing.get(phase) or {}).get("duration", 0.0)
+
+    tool_summary = ", ".join(f"{name}: {n}" for name, n in tools.most_common())
+    metric_summary = ", ".join(
+        f"{name}: {format_numeric(value)}" for name, value in metrics.items()
+    )
+    parts = [
+        f"Finished example_id={state.get('example_id')}",
+        f"trajectory_id={state.get('trajectory_id')}",
+        f"tools=[{tool_summary}]",
+        "timing="
+        + format_timing_plain(
+            setup=duration("setup"),
+            generation=duration("generation"),
+            scoring=duration("scoring"),
+            overhead=timing.get("overhead", 0.0),
+            model=duration("model"),
+            env=duration("env"),
+        ),
+        f"stop={state.get('stop_condition')}",
+        f"reward={format_numeric(state.get('reward') or 0.0)}",
+        "metrics={" + metric_summary + "}",
+    ]
+    if state.get("error"):
+        parts.append(f"error={truncate(state['error']['error_chain_str'], 200)}")
+    if state.get("is_truncated"):
+        parts.append("truncated=True")
+    logger.info(" | ".join(parts))