verifiers 0.4.1.dev65__tar.gz → 0.4.1.dev68__tar.gz

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev65
+Version: 0.4.1.dev68
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/tests/integration/fixtures/counter_tool.py

@@ -35,6 +35,11 @@ class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
         return float(self.config.constant_reward or trace.state.count >= 2)
 
 
+def any_response(response: vf.Response) -> bool:
+    """A plugged stop that ends the run on its first model response."""
+    return True
+
+
 class CounterConfig(vf.TasksetConfig):
     task: CounterTaskConfig = CounterTaskConfig()
 

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/tests/integration/test_limits.py

@@ -92,3 +92,25 @@ async def test_max_turns(run, harness, harness_runtime):
     assert trace.num_branches == 1
     assert trace.stop_condition == "max_turns", [e.message for e in trace.errors]
     assert trace.num_turns == 1
+
+
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_stop(run, scripted_model, skip):
+    stop = {"fn": "counter_tool:any_response", "skip_scoring": skip}
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "subprocess"},
+        client=scripted_model,
+        taskset_overrides={
+            "task": {"constant_reward": True, "stops": {"any_response": stop}}
+        },
+    )
+    (trace,) = episode.traces
+    assert trace.stop_condition == "any_response" and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "any_response"
+    else:
+        assert trace.reward == 1
+        assert "scoring_skipped" not in trace.info

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/tests/unit/test_task.py

@@ -36,6 +36,10 @@ class HookTask(vf.Task[HookData]):
     async def single_turn(self, trace: vf.Trace) -> bool:
         return trace.num_turns >= 1
 
+    @vf.stop(skip_scoring=True)
+    async def no_reply(self, trace: vf.Trace) -> bool:
+        return trace.last_reply is None
+
     @vf.reward
     async def lcs(self, trace: vf.Trace) -> float:
         return 0.5
@@ -49,7 +53,7 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
     fns = tmp_path / "fns.py"
     fns.write_text(PLUGGED_FNS_PY)
     config = vf.TaskConfig(
-        stops={"single_turn": vf.DecoratedFunctionConfig(fn=f"{fns}:two_turns")},
+        stops={"single_turn": vf.StopFunctionConfig(fn=f"{fns}:two_turns")},
         metrics={"reply_length": vf.DecoratedFunctionConfig(fn=f"{fns}:reply_length")},
         rewards={
             "exact_match": vf.RewardFunctionConfig(fn=f"{fns}:exact_match", weight=0.5),
@@ -70,9 +74,10 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
     )
 
     # the plugged `single_turn` replaces the decorated one: no stop after one turn
-    (stop,) = task.hooks("stop")
+    no_reply, stop = task.hooks("stop")
     assert stop.__name__ == "single_turn"
     assert not await stop(trace)
+    assert no_reply.stop_skip_scoring and not stop.stop_skip_scoring
 
     await task.score(trace)
     # `lcs` scored by the plugged marker (config wins the name clash), config weight
@@ -102,3 +107,27 @@ async def test_defer_scoring_defers_only_task_scoring() -> None:
     assert trace.rewards == {}
     await task.score(trace)
     assert set(trace.rewards) == {"fmt", "lcs"}
+
+
+def test_plugged_stops_set_skip_scoring(tmp_path) -> None:
+    fns = tmp_path / "fns.py"
+    fns.write_text(PLUGGED_FNS_PY)
+    data = HookData(idx=0, prompt="abc")
+
+    def skips(stops: dict) -> dict[str, bool]:
+        config = vf.TaskConfig(stops=stops)
+        return {
+            fn.__name__: fn.stop_skip_scoring
+            for fn in HookTask(data, config).hooks("stop")
+        }
+
+    # a fresh plugged stop scores by default; a fn-less entry keeps or overrides the decorator's
+    assert skips({"two": vf.StopFunctionConfig(fn=f"{fns}:two_turns")})["two"] is False
+    assert skips(
+        {"two": vf.StopFunctionConfig(fn=f"{fns}:two_turns", skip_scoring=True)}
+    )["two"]
+    assert skips({"no_reply": vf.StopFunctionConfig()})["no_reply"] is True
+    assert (
+        skips({"no_reply": vf.StopFunctionConfig(skip_scoring=False)})["no_reply"]
+        is False
+    )

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/verifiers/v1/__init__.py

@@ -56,6 +56,7 @@ if TYPE_CHECKING:
     from verifiers.v1.configs.task import (
         DecoratedFunctionConfig,
         RewardFunctionConfig,
+        StopFunctionConfig,
         TaskConfig,
     )
     from verifiers.v1.configs.taskset import SharedTasksetConfig, TasksetConfig
@@ -301,6 +302,7 @@ _NAME_TO_MODULE = {
     "State": "verifiers.v1.state",
     "StateT": "verifiers.v1.state",
     "StaticPoolConfig": "verifiers.v1.configs.serve",
+    "StopFunctionConfig": "verifiers.v1.configs.task",
     "SubprocessConfig": "verifiers.v1.configs.runtime",
     "SystemMessage": "verifiers.v1.types",
     "TRACE_VERSION": "verifiers.v1.trace",
@@ -480,6 +482,7 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "SharedTasksetConfig",
     "DecoratedFunctionConfig",
     "RewardFunctionConfig",
+    "StopFunctionConfig",
     "BaseConfig",
     "Harness",
     "HarnessSession",

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/verifiers/v1/configs/task.py

@@ -14,13 +14,21 @@ class DecoratedFunctionConfig(BaseConfig):
     fn: str | None = None
     """Import path to the function: `pkg.module.function`, `pkg.module:function`,
     or `path/to/file.py:function`. Unset keeps the task's decorated method with the
-    entry's name and only overrides its metadata (priority, weight)."""
+    entry's name and only overrides its metadata (priority, weight, skip_scoring)."""
 
     priority: int | None = None
     """Execution order, like the decorator's — higher runs first, ties break by name.
     Unset keeps the replaced method's priority (0 for a fresh function)."""
 
 
+class StopFunctionConfig(DecoratedFunctionConfig):
+    """A plugged function standing in for a `@vf.stop` task method."""
+
+    skip_scoring: bool | None = None
+    """Skip task scoring for a trace this stop ends: it keeps no rewards, so its reward
+    is 0. Unset keeps the replaced method's setting (false for a fresh function)."""
+
+
 class RewardFunctionConfig(DecoratedFunctionConfig):
     """A plugged function standing in for a `@vf.reward` task method."""
 
@@ -39,7 +47,7 @@ class TaskConfig(BaseConfig):
     judges: Judges = Field(default_factory=list)
     """Judge plugins run after task rewards, set through `--env.taskset.task.judges`."""
 
-    stops: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
+    stops: dict[str, StopFunctionConfig] = Field(default_factory=dict)
     """Typed `Request`, `Response`, or `Trace` stop predicates plugged by name and
     merged with the task's `@vf.stop` methods (a plugged function replaces a
     decorated one with the same name)."""

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/verifiers/v1/rollout.py

@@ -157,6 +157,9 @@ class Rollout:
             for fn in discover_decorated(task, "intercept")
         ]
         stops = [(hook_boundary(fn, allow_trace=True), fn) for fn in task.hooks("stop")]
+        self._skip_scoring_stops = {
+            fn.__name__ for _, fn in stops if getattr(fn, "stop_skip_scoring", False)
+        }
         execution_policy = runtime_config
         self._session = RolloutSession(
             ctx=ctx,
@@ -722,11 +725,14 @@ class Rollout:
         )
 
     def _scoring_skip_reason(self) -> str | None:
-        """Why the resolved scoring policy skips task scoring for this trace, if it does."""
+        """Why the resolved scoring policy, or the `skip_scoring` stop that ended this trace,
+        skips task scoring for it, if either does."""
         if self._scoring_policy.skip_on_truncated and self.trace.is_truncated:
             return "truncated"
         if self._scoring_policy.skip_on_timeout and self.trace.is_timeout:
             return "timeout"
+        if self.trace.stop_condition in self._skip_scoring_stops:
+            return self.trace.stop_condition
         return None
 
     def _metrics_deadline(self) -> float:

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/verifiers/v1/utils/decorators.py

@@ -83,12 +83,18 @@ def tool(func: F | None = None, name: str | None = None) -> F | Callable[[F], F]
 
 
 @overload
-def stop(func: F, priority: int = 0) -> F: ...
+def stop(func: F, priority: int = 0, skip_scoring: bool = False) -> F: ...
 @overload
-def stop(func: None = None, priority: int = 0) -> Callable[[F], F]: ...
-def stop(func: F | None = None, priority: int = 0) -> F | Callable[[F], F]:
-    """Stop when a typed `Request`, `Response`, or `Trace` predicate returns true."""
-    decorator = mark("stop", stop_priority=priority)
+def stop(
+    func: None = None, priority: int = 0, skip_scoring: bool = False
+) -> Callable[[F], F]: ...
+def stop(
+    func: F | None = None, priority: int = 0, skip_scoring: bool = False
+) -> F | Callable[[F], F]:
+    """Stop when a typed `Request`, `Response`, or `Trace` predicate returns true. With
+    `skip_scoring`, a trace this stop ends skips task scoring: it keeps no rewards, so
+    its reward is 0."""
+    decorator = mark("stop", stop_priority=priority, stop_skip_scoring=skip_scoring)
     return decorator if func is None else decorator(func)
 
 

{verifiers-0.4.1.dev65 → verifiers-0.4.1.dev68}/verifiers/v1/utils/loaders.py

@@ -15,7 +15,11 @@ from pydantic_config import BaseConfig
 from verifiers.v1.configs.env import EnvConfig
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig
-from verifiers.v1.configs.task import DecoratedFunctionConfig, RewardFunctionConfig
+from verifiers.v1.configs.task import (
+    DecoratedFunctionConfig,
+    RewardFunctionConfig,
+    StopFunctionConfig,
+)
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv
@@ -279,6 +283,11 @@ def load_plugged_fn(
         if weight is None:
             weight = getattr(fn, "_vf_weight", 1.0)
         plugged._vf_weight = weight
+    if isinstance(config, StopFunctionConfig):
+        skip_scoring = config.skip_scoring
+        if skip_scoring is None:
+            skip_scoring = getattr(fn, "stop_skip_scoring", False)
+        plugged.stop_skip_scoring = skip_scoring
     return plugged