verifiers 0.3.1.dev15__tar.gz → 0.3.1.dev17__tar.gz

{verifiers-0.3.1.dev15 → verifiers-0.3.1.dev17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev15
+Version: 0.3.1.dev17
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.1.dev15 → verifiers-0.3.1.dev17}/tests/v1/test_scoring.py

@@ -1,3 +1,5 @@
+import pytest
+
 import verifiers.v1 as vf
 from verifiers.v1.graph import MessageNode
 from verifiers.v1.types import AssistantMessage, UserMessage
@@ -33,6 +35,10 @@ class HookTask(vf.Task[HookData]):
     async def lcs(self, trace: vf.Trace) -> float:
         return 0.5
 
+    @vf.reward(weight=0.0)
+    async def fmt(self, trace: vf.Trace) -> float:
+        return 1.0
+
 
 async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
     fns = tmp_path / "fns.py"
@@ -43,6 +49,7 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
         rewards={
             "exact_match": vf.RewardFunctionConfig(fn=f"{fns}:exact_match", weight=0.5),
             "lcs": vf.RewardFunctionConfig(fn=f"{fns}:marker", weight=2.0),
+            "fmt": vf.RewardFunctionConfig(weight=1.0),
         },
     )
     task = HookTask(HookData(idx=0, prompt="abc", answer="cba"), config)
@@ -69,6 +76,14 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
     assert trace.rewards["exact_match"].score == 1.0
     assert trace.rewards["exact_match"].weight == 0.5
     assert trace.metrics["reply_length"] == 3.0
+    # `fmt` keeps the decorated body, only its weight is overridden (0.0 -> 1.0)
+    assert trace.rewards["fmt"].score == 1.0
+    assert trace.rewards["fmt"].weight == 1.0
+
+    # a fn-less entry must name an existing decorated method
+    config = vf.TaskConfig(rewards={"nope": vf.RewardFunctionConfig(weight=1.0)})
+    with pytest.raises(ValueError, match="no @vf.reward method named 'nope'"):
+        HookTask(HookData(idx=0, prompt="abc"), config).hooks("reward")
 
 
 def test_compare_stdout_results_accepts_token_equal_text() -> None:

{verifiers-0.3.1.dev15 → verifiers-0.3.1.dev17}/verifiers/v1/__init__.py

@@ -27,7 +27,6 @@ from verifiers.v1.configs.serve import (
 )
 from verifiers.v1.configs.task import (
     DecoratedFunctionConfig,
-    FunctionConfig,
     RewardFunctionConfig,
     TaskConfig,
 )
@@ -244,7 +243,6 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Taskset",
     "TaskConfig",
     "TasksetConfig",
-    "FunctionConfig",
     "DecoratedFunctionConfig",
     "RewardFunctionConfig",
     "BaseConfig",

{verifiers-0.3.1.dev15 → verifiers-0.3.1.dev17}/verifiers/v1/configs/task.py

@@ -8,25 +8,24 @@ from pydantic_config import BaseConfig
 from verifiers.v1.configs.judge import Judges, check_judges, resolve_judges
 
 
-class FunctionConfig(BaseConfig):
-    """A function plugged in by import path."""
+class DecoratedFunctionConfig(BaseConfig):
+    """A plugged function standing in for a decorated task method."""
 
-    fn: str
+    fn: str | None = None
     """Import path to the function: `pkg.module.function`, `pkg.module:function`,
-    or `path/to/file.py:function`."""
-
-
-class DecoratedFunctionConfig(FunctionConfig):
-    """A plugged function standing in for a decorated task method."""
+    or `path/to/file.py:function`. Unset keeps the task's decorated method with the
+    entry's name and only overrides its metadata (priority, weight)."""
 
-    priority: int = 0
-    """Execution order, like the decorator's — higher runs first, ties break by name."""
+    priority: int | None = None
+    """Execution order, like the decorator's — higher runs first, ties break by name.
+    Unset keeps the replaced method's priority (0 for a fresh function)."""
 
 
 class RewardFunctionConfig(DecoratedFunctionConfig):
     """A plugged function standing in for a `@vf.reward` task method."""
 
-    weight: FiniteFloat = 1.0
+    weight: FiniteFloat | None = None
+    """Unset keeps the replaced method's weight (1.0 for a fresh function)."""
 
 
 class TaskConfig(BaseConfig):

{verifiers-0.3.1.dev15 → verifiers-0.3.1.dev17}/verifiers/v1/task.py

@@ -233,8 +233,8 @@ class Task(Generic[DataT, StateT, ConfigT]):
 
     def hooks(self, attr: str) -> list[Callable[..., Any]]:
         """The task's `@vf.<attr>` methods merged with the config-plugged functions —
-        a plugged function replaces a decorated method with the same name — sorted by
-        descending priority then name."""
+        a plugged function replaces a decorated method with the same name (a `fn`-less
+        entry only overrides its metadata) — sorted by descending priority then name."""
         from verifiers.v1.utils.loaders import load_plugged_fn
 
         plugged = {
@@ -244,7 +244,8 @@ class Task(Generic[DataT, StateT, ConfigT]):
         }[attr]
         merged = {fn.__name__: fn for fn in discover_decorated(self, attr)}
         merged |= {
-            name: load_plugged_fn(name, spec, attr) for name, spec in plugged.items()
+            name: load_plugged_fn(name, spec, attr, decorated=merged.get(name))
+            for name, spec in plugged.items()
         }
         fns = list(merged.values())
         fns.sort(key=lambda fn: (-getattr(fn, f"{attr}_priority", 0), fn.__name__))

{verifiers-0.3.1.dev15 → verifiers-0.3.1.dev17}/verifiers/v1/utils/loaders.py

@@ -244,11 +244,24 @@ def plugged_fn(path: str) -> Callable[..., Any]:
 
 
 def load_plugged_fn(
-    name: str, config: DecoratedFunctionConfig, attr: str
+    name: str,
+    config: DecoratedFunctionConfig,
+    attr: str,
+    decorated: Callable[..., Any] | None = None,
 ) -> Callable[..., Any]:
-    """Build a task hook from a config entry: the imported async function wrapped
-    under the plugged `name`, tagged like its `@vf.<attr>` equivalent."""
-    fn = plugged_fn(config.fn)
+    """Build a task hook from a config entry: the imported async function — or, with
+    no `fn`, the task's `decorated` method it overrides — wrapped under the plugged
+    `name`, tagged like its `@vf.<attr>` equivalent. Unset metadata fields keep the
+    wrapped function's tags."""
+    if config.fn:
+        fn = plugged_fn(config.fn)
+    elif decorated is not None:
+        fn = decorated
+    else:
+        raise ValueError(
+            f"{attr} {name!r} plugs no `fn` and the task has no @vf.{attr} "
+            f"method named {name!r} to override"
+        )
 
     @functools.wraps(fn)
     def plugged(*args: Any, **kwargs: Any) -> Any:
@@ -256,9 +269,15 @@ def load_plugged_fn(
 
     plugged.__name__ = name
     setattr(plugged, attr, True)
-    setattr(plugged, f"{attr}_priority", config.priority)
+    priority = config.priority
+    if priority is None:
+        priority = getattr(fn, f"{attr}_priority", 0)
+    setattr(plugged, f"{attr}_priority", priority)
     if isinstance(config, RewardFunctionConfig):
-        plugged._vf_weight = config.weight
+        weight = config.weight
+        if weight is None:
+            weight = getattr(fn, "_vf_weight", 1.0)
+        plugged._vf_weight = weight
     return plugged