verifiers 0.1.15.dev151__tar.gz → 0.1.15.dev153__tar.gz

{verifiers-0.1.15.dev151 → verifiers-0.1.15.dev153}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev151
+Version: 0.1.15.dev153
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev151 → verifiers-0.1.15.dev153}/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py

@@ -34,6 +34,7 @@ from verifiers.envs.experimental.composable import SandboxSpec, SandboxTaskSet
 logger = logging.getLogger(__name__)
 
 DATASET_NAME = "AweAI-Team/Scale-SWE"
+REGISTRY_PREFIX = "us-central1-docker.pkg.dev/prime-intellect-platform/prod-sandbox"
 
 ENV_VARS_SCALE_SWE = {
     "PATH": (
@@ -70,6 +71,13 @@ if __name__ == "__main__":
 """
 
 
+def _build_docker_image(info: dict) -> str:
+    image = info["image_url"]
+    if image.startswith(REGISTRY_PREFIX):
+        return image
+    return f"{REGISTRY_PREFIX}/{image}"
+
+
 class ScaleSWERubric(vf.Rubric):
     """Scores Scale-SWE tasks by checking merged F2P/P2P pytest results."""
 
@@ -155,7 +163,7 @@ class ScaleSWETaskSet(SandboxTaskSet):
 
     def get_sandbox_spec(self, info: dict) -> SandboxSpec:
         return SandboxSpec(
-            image=info["image_url"],
+            image=_build_docker_image(info),
             timeout_minutes=self.timeout_minutes,
         )
 

{verifiers-0.1.15.dev151 → verifiers-0.1.15.dev153}/verifiers/utils/display_utils.py

@@ -533,6 +533,25 @@ def _timing_parts(
     return parts
 
 
+def format_timing_plain(
+    setup: float = 0.0,
+    generation: float = 0.0,
+    scoring: float = 0.0,
+    overhead: float = 0.0,
+    model: float = 0.0,
+    env: float = 0.0,
+) -> str:
+    """Return a plain-text compact timing breakdown (no Rich styling)."""
+    parts = _timing_parts(setup, generation, scoring, overhead, model, env)
+    segments: list[str] = []
+    for label, value, subs in parts:
+        segment = f"{label} {value}"
+        if subs:
+            segment += " (" + " + ".join(f"{sl} {sv}" for sl, sv in subs) + ")"
+        segments.append(segment)
+    return " + ".join(segments)
+
+
 def format_timing_rich(
     setup: float = 0.0,
     generation: float = 0.0,

{verifiers-0.1.15.dev151 → verifiers-0.1.15.dev153}/verifiers/v1/harness.py

@@ -71,6 +71,7 @@ from .utils.sandbox_program_utils import (
     python_program_sandbox,
     run_sandbox_python_program,
 )
+from .utils.logging_utils import log_rollout_finish, log_rollout_start
 from .utils.prompt_utils import (
     SystemPrompt,
     SystemPromptStrategy,
@@ -241,6 +242,7 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
 
     async def run(self, task: Task, state: State | None = None) -> State:
         state = await self.init_state(task) if state is None else state
+        log_rollout_start(state)
         timing_recorded = False
         completed = False
         try:
@@ -272,6 +274,7 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
                     state.strip_runtime_handles()
             elif completed:
                 state.assert_serializable()
+            log_rollout_finish(state)
         return state
 
     def record_error(self, state: State, error: Error) -> None:

verifiers-0.1.15.dev153/verifiers/v1/utils/logging_utils.py

@@ -0,0 +1,57 @@
+import logging
+from collections import Counter
+
+from verifiers.utils.display_utils import format_numeric, format_timing_plain
+from verifiers.utils.logging_utils import truncate
+
+from ..state import State
+
+logger = logging.getLogger("verifiers.v1.rollout")
+
+
+def log_rollout_start(state: State) -> None:
+    logger.info(
+        f"Started example_id={state.get('example_id')} "
+        f"| trajectory_id={state.get('trajectory_id')}"
+    )
+
+
+def log_rollout_finish(state: State) -> None:
+    tools = Counter(
+        call["name"]
+        for step in state.get("trajectory") or []
+        for message in step.get("completion") or []
+        for call in message.get("tool_calls") or []
+    )
+    timing = state.get("timing") or {}
+    metrics = state.get("metrics") or {}
+
+    def duration(phase: str) -> float:
+        return (timing.get(phase) or {}).get("duration", 0.0)
+
+    tool_summary = ", ".join(f"{name}: {n}" for name, n in tools.most_common())
+    metric_summary = ", ".join(
+        f"{name}: {format_numeric(value)}" for name, value in metrics.items()
+    )
+    parts = [
+        f"Finished example_id={state.get('example_id')}",
+        f"trajectory_id={state.get('trajectory_id')}",
+        f"tools=[{tool_summary}]",
+        "timing="
+        + format_timing_plain(
+            setup=duration("setup"),
+            generation=duration("generation"),
+            scoring=duration("scoring"),
+            overhead=timing.get("overhead", 0.0),
+            model=duration("model"),
+            env=duration("env"),
+        ),
+        f"stop={state.get('stop_condition')}",
+        f"reward={format_numeric(state.get('reward') or 0.0)}",
+        "metrics={" + metric_summary + "}",
+    ]
+    if state.get("error"):
+        parts.append(f"error={truncate(state['error']['error_chain_str'], 200)}")
+    if state.get("is_truncated"):
+        parts.append("truncated=True")
+    logger.info(" | ".join(parts))