verifiers 0.2.1.dev35__tar.gz → 0.2.1.dev37__tar.gz

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev35
+Version: 0.2.1.dev37
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/tests/v1/fixtures/echo_user_sim_v1.py

@@ -35,7 +35,7 @@ class EchoUserSimUser(vf.User[vf.UserConfig, EchoUserSimState]):
         if self.turns >= len(self.phrases):
             self.state.user_finished = True
             return []
-        return [{"role": "user", "content": self.phrases[self.turns]}]
+        return [vf.UserMessage(content=self.phrases[self.turns])]
 
 
 class EchoUserSimData(vf.TaskData):

verifiers-0.2.1.dev37/tests/v1/test_color_codeword_tasks.py

@@ -0,0 +1,18 @@
+from color_codeword_v1 import taskset
+
+
+def test_color_images_are_rendered_once(monkeypatch) -> None:
+    calls: list[tuple[int, int, int]] = []
+    render = taskset.image_data_url
+
+    def counted(image) -> str:
+        calls.append(image.getpixel((0, 0)))
+        return render(image)
+
+    monkeypatch.setattr(taskset, "image_data_url", counted)
+    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
+        num_tasks=100
+    )
+
+    assert len(tasks) == 100
+    assert calls == list(taskset.COLOR_RGB.values())

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/verifiers/v1/cli/dashboard/eval.py

@@ -448,10 +448,8 @@ def Rows(groups: list[list[Rollout]], now: float, runtime_type: str) -> Table:
                         stop = f"{stop} (truncated)".strip()
             else:
                 state, result, stop = rollout.phase, "", ""
-            descriptor = (
-                rollout.runtime.descriptor if rollout.runtime is not None else None
-            )
-            runtime = f"{runtime_type}({descriptor})" if descriptor else runtime_type
+            runtime_id = t.runtime.id if t.runtime is not None else None
+            runtime = f"{runtime_type}({runtime_id})" if runtime_id else runtime_type
             turns = t.num_turns
             start = t.timing.boot.start or t.timing.setup.start
             end = (

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/verifiers/v1/cli/debug.py

@@ -63,14 +63,6 @@ def task_info(task) -> dict[str, Any]:
     return {"idx": task.data.idx, "name": task.data.name, "workdir": task.data.workdir}
 
 
-def runtime_info(runtime: Runtime) -> dict[str, Any]:
-    return {
-        "type": runtime.type,
-        "name": runtime.name,
-        "descriptor": runtime.descriptor,
-    }
-
-
 def result_info(result: ProgramResult, start: float) -> dict[str, Any]:
     ok = result.exit_code == 0
     return {
@@ -132,7 +124,6 @@ def capture_trace_error(trace: Trace, error: BaseException) -> None:
 def record_debug_error(
     trace: Trace,
     debug: dict[str, Any],
-    runtime: Runtime,
     error: BaseException,
     setup_timeout: float | None,
     action_timeout: float | None,
@@ -152,7 +143,6 @@ def record_debug_error(
         else trace.timing.setup.start or trace.timing.boot.start
     )
     debug.update(error_info(error, error_start, timeout, stage))
-    debug.setdefault("runtime", runtime_info(runtime))
     capture_trace_error(trace, error)
 
 
@@ -223,6 +213,7 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         resolve_runtime_config(config.runtime, task),
         name=f"debug-{task.data.idx}-{uuid4().hex[:8]}",
     )
+    trace.runtime = runtime.info
     setup_timeout = (
         config.timeout.setup
         if config.timeout.setup is not None
@@ -234,7 +225,6 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         now = time.time()
         trace.timing.boot.end = now
         trace.timing.setup.start = now
-        debug["runtime"] = runtime_info(runtime)
         await asyncio.wait_for(
             invoke(task.setup, {"trace": trace, "runtime": runtime}),
             setup_timeout,
@@ -249,13 +239,9 @@ async def debug_task(task: Task, config: DebugConfig) -> tuple[Trace, bool]:
         trace.stop(str(debug["reason"]))
     except asyncio.CancelledError as e:
         cancelled = True
-        record_debug_error(
-            trace, debug, runtime, e, setup_timeout, config.timeout.total
-        )
+        record_debug_error(trace, debug, e, setup_timeout, config.timeout.total)
     except Exception as e:
-        record_debug_error(
-            trace, debug, runtime, e, setup_timeout, config.timeout.total
-        )
+        record_debug_error(trace, debug, e, setup_timeout, config.timeout.total)
     finally:
         trace.info["debug"] = debug
         try:

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/verifiers/v1/cli/eval/resume.py

@@ -59,7 +59,6 @@ class Finished(Rollout):
         self.trace = trace
         self.task = Task(trace.task.data)
         self.phase = Phase.DONE
-        self.runtime = None
 
 
 def load(

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/verifiers/v1/runtimes/base.py

@@ -116,10 +116,6 @@ class Runtime(ABC):
     def type(self) -> str:
         return self.config.type
 
-    @property
-    def descriptor(self) -> str | None:
-        return self.info.id
-
     @abstractmethod
     async def start(self) -> None:
         pass

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/verifiers/v1/runtimes/modal.py

@@ -187,7 +187,7 @@ class ModalRuntime(Runtime):
         # the sandbox via Modal's sync API so the costly resource isn't left to its max-lifetime.
         # Idempotent — the async `stop` handles the normal path, and a second terminate is a no-op.
         sandbox, self._sandbox = self._sandbox, None
-        if sandbox is not None:  # `info.id` kept so descriptor survives teardown
+        if sandbox is not None:  # keep info.id available after teardown
             with contextlib.suppress(Exception):
                 sandbox.terminate()
 

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/verifiers/v1/runtimes/prime.py

@@ -257,9 +257,7 @@ class PrimeRuntime(Runtime):
         client, self._client = self._client, None  # `_client` is the idempotency guard
         if client is None:
             return
-        if (
-            self.info.id is not None
-        ):  # kept (not nulled) so descriptor survives teardown
+        if self.info.id is not None:  # keep info.id available after teardown
             try:
                 await client.delete(self.info.id)
             except Exception as e:

{verifiers-0.2.1.dev35 → verifiers-0.2.1.dev37}/verifiers/v1/tasksets/textarena/taskset.py

@@ -75,9 +75,9 @@ class TextArenaUser(vf.User[vf.UserConfig, TextArenaState]):
             with open(OUTCOME_FILE, "w") as f:
                 json.dump({"reward": reward, "reason": reason}, f)
             self.state.game_over = True
-            return [{"role": "user", "content": reason}]
+            return [vf.UserMessage(content=reason)]
         _, observation = env.get_observation()
-        return [{"role": "user", "content": self._latest_feedback(str(observation))}]
+        return [vf.UserMessage(content=self._latest_feedback(str(observation)))]
 
 
 class TextArenaTaskConfig(vf.TaskConfig):

verifiers-0.2.1.dev37/verifiers/v1/utils/image.py

@@ -0,0 +1,9 @@
+import base64
+from io import BytesIO
+
+
+def image_data_url(image) -> str:
+    """Encode a PIL-compatible image as a base64 PNG data URL."""
+    buffer = BytesIO()
+    image.save(buffer, format="PNG")
+    return f"data:image/png;base64,{base64.b64encode(buffer.getvalue()).decode()}"

verifiers-0.2.1.dev35/tests/v1/test_color_codeword_tasks.py

@@ -1,18 +0,0 @@
-from color_codeword_v1 import taskset
-
-
-def test_color_images_are_rendered_once(monkeypatch) -> None:
-    calls: list[str] = []
-    render = taskset.color_data_url
-
-    def counted(color: str, size: int = 100) -> str:
-        calls.append(color)
-        return render(color, size)
-
-    monkeypatch.setattr(taskset, "color_data_url", counted)
-    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
-        num_tasks=100
-    )
-
-    assert len(tasks) == 100
-    assert calls == list(taskset.COLOR_MAP)