verifiers 0.1.15.dev411__tar.gz → 0.1.15.dev413__tar.gz

{verifiers-0.1.15.dev411 → verifiers-0.1.15.dev413}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev411
+Version: 0.1.15.dev413
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev411 → verifiers-0.1.15.dev413}/verifiers/v1/cli/validate.py

@@ -29,9 +29,12 @@ from verifiers.v1.cli.resolve import (
     with_positional_taskset,
 )
 from verifiers.v1.configs.validate import ValidateConfig
+from verifiers.v1.decorators import invoke
 from verifiers.v1.env import resolve_runtime_config
 from verifiers.v1.runtimes import make_runtime
+from verifiers.v1.state import state_cls
 from verifiers.v1.taskset import Taskset
+from verifiers.v1.trace import Trace
 
 logger = logging.getLogger(__name__)
 
@@ -81,8 +84,12 @@ async def _validate_task(taskset: Taskset, task, config: ValidateConfig) -> dict
     )
     valid, exc = False, None
     try:
+        trace = Trace(task=task, state=state_cls(type(taskset))())
         await runtime.start()
-        await asyncio.wait_for(taskset.setup(task, runtime), setup_timeout)
+        await asyncio.wait_for(
+            invoke(taskset.setup, {"task": task, "trace": trace, "runtime": runtime}),
+            setup_timeout,
+        )
         valid = await asyncio.wait_for(
             taskset.validate(task, runtime), config.validate_timeout
         )

{verifiers-0.1.15.dev411 → verifiers-0.1.15.dev413}/verifiers/v1/harnesses/rlm/harness.py

@@ -144,10 +144,10 @@ class RLMHarness(Harness[RLMHarnessConfig]):
 
     @metric
     async def rlm(self, trace: Trace, runtime: Runtime) -> dict[str, float]:
-        # rlm writes a session meta.json with a rich `metrics` block (turns, token
-        # stats, compactions, tool-call counts). There's one top-level session dir
-        # (sub-harnesses nest as sub-*/), so the glob matches a single file. Surface
-        # its numeric metrics under an `rlm_` prefix; non-numeric fields (e.g.
+        # rlm writes a session meta.json with a rich `metrics` block (compactions,
+        # ipython input size, programmatic tool-call counts). There's one top-level
+        # session dir (sub-harnesses nest as sub-*/), so the glob matches a single
+        # file. Surface its numeric metrics as-is; non-numeric fields (e.g.
         # stop_reason) don't fit the float-only trace metrics, so they're skipped.
         result = await runtime.run(
             ["sh", "-c", f"cat {RLM_HOME}/sessions/*/meta.json"], {}
@@ -159,7 +159,7 @@ class RLMHarness(Harness[RLMHarnessConfig]):
         except json.JSONDecodeError:
             return {}
         return {
-            f"rlm_{key}": float(value)
+            key: float(value)
             for key, value in meta.get("metrics", {}).items()
             if isinstance(value, (int, float)) and not isinstance(value, bool)
         }

{verifiers-0.1.15.dev411 → verifiers-0.1.15.dev413}/verifiers/v1/rollout.py

@@ -20,7 +20,7 @@ from enum import StrEnum
 
 from verifiers.v1.harness import Harness
 from verifiers.v1.clients import RolloutContext
-from verifiers.v1.decorators import discover_decorated
+from verifiers.v1.decorators import discover_decorated, invoke
 from verifiers.v1.errors import (
     HarnessError,
     RolloutError,
@@ -165,7 +165,10 @@ class Rollout:
                 boundary(TasksetError, "taskset setup"),
                 asyncio.timeout_at(setup_deadline),
             ):
-                await self.taskset.setup(self.task, runtime)
+                await invoke(
+                    self.taskset.setup,
+                    {"task": self.task, "trace": trace, "runtime": runtime},
+                )
             async with (
                 boundary(HarnessError, "harness setup"),
                 asyncio.timeout_at(setup_deadline),

{verifiers-0.1.15.dev411 → verifiers-0.1.15.dev413}/verifiers/v1/taskset.py

@@ -80,10 +80,16 @@ class Taskset(Generic[TaskT, ConfigT, StateT]):
         multi-turn conversation (e.g. a TextArena game)."""
         return None
 
-    async def setup(self, task: TaskT, runtime: Runtime) -> None:
+    async def setup(self, task: TaskT, trace: Trace, runtime: Runtime) -> None:
         """Prepare the live runtime for this task, after `runtime.start()` and before the
         harness runs. No-op by default; override to run per-task setup in the runtime (e.g.
-        a SWE row checking out its base commit). Errors propagate and fail the rollout."""
+        a SWE row checking out its base commit). Errors propagate and fail the rollout.
+
+        Like the scoring hooks, `setup` declares the inputs it needs *by parameter name* and
+        the framework injects them: any subset of `task`, `trace`, `runtime`. The trace (and
+        its per-rollout `trace.state`) already exists when `setup` runs, so an override may
+        stash per-rollout state there — e.g. `setup(self, task, trace, runtime)` or the
+        legacy `setup(self, task, runtime)` both work."""
         return None
 
     async def finalize(self, task: TaskT, trace: Trace, runtime: Runtime) -> None: