verifiers 0.3.1.dev9__tar.gz → 0.3.1.dev11__tar.gz

{verifiers-0.3.1.dev9 → verifiers-0.3.1.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev9
+Version: 0.3.1.dev11
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

verifiers-0.3.1.dev11/tests/v1/test_scoring.py

@@ -0,0 +1,101 @@
+import verifiers.v1 as vf
+from verifiers.v1.graph import MessageNode
+from verifiers.v1.types import AssistantMessage, UserMessage
+
+PLUGGED_FNS_PY = """
+async def reply_length(trace) -> float:
+    return float(len(trace.last_reply or ""))
+
+
+async def exact_match(task, trace) -> float:
+    return float(task.answer in (trace.last_reply or ""))
+
+
+async def marker(trace) -> float:
+    return 0.125
+
+
+async def two_turns(trace) -> bool:
+    return trace.num_turns >= 2
+"""
+
+
+class HookData(vf.TaskData):
+    answer: str = ""
+
+
+class HookTask(vf.Task[HookData]):
+    @vf.stop
+    async def single_turn(self, trace: vf.Trace) -> bool:
+        return trace.num_turns >= 1
+
+    @vf.reward
+    async def lcs(self, trace: vf.Trace) -> float:
+        return 0.5
+
+
+async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
+    fns = tmp_path / "fns.py"
+    fns.write_text(PLUGGED_FNS_PY)
+    config = vf.TaskConfig(
+        stops={"single_turn": vf.DecoratedFunctionConfig(fn=f"{fns}:two_turns")},
+        metrics={"reply_length": vf.DecoratedFunctionConfig(fn=f"{fns}:reply_length")},
+        rewards={
+            "exact_match": vf.RewardFunctionConfig(fn=f"{fns}:exact_match", weight=0.5),
+            "lcs": vf.RewardFunctionConfig(fn=f"{fns}:marker", weight=2.0),
+        },
+    )
+    task = HookTask(HookData(idx=0, prompt="abc", answer="cba"), config)
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="HookTask", data=task.data),
+        nodes=[
+            MessageNode(parent=None, message=UserMessage(content="abc"), sampled=False),
+            MessageNode(
+                parent=0, message=AssistantMessage(content="cba"), sampled=True
+            ),
+        ],
+    )
+
+    # the plugged `single_turn` replaces the decorated one: no stop after one turn
+    (stop,) = task.hooks("stop")
+    assert stop.__name__ == "single_turn"
+    assert not await stop(trace)
+
+    await task.score(trace)
+    # `lcs` scored by the plugged marker (config wins the name clash), config weight
+    assert trace.rewards["lcs"].score == 0.125
+    assert trace.rewards["lcs"].weight == 2.0
+    assert trace.rewards["exact_match"].score == 1.0
+    assert trace.rewards["exact_match"].weight == 0.5
+    assert trace.metrics["reply_length"] == 3.0
+
+
+def test_compare_stdout_results_accepts_token_equal_text() -> None:
+    assert vf.compare_stdout_results("hello   world\n", "hello world\n")
+
+
+def test_compare_stdout_results_keeps_numeric_tolerance() -> None:
+    assert vf.compare_stdout_results("1.0001 2.0\n", "1.0002 2.0009\n")
+
+
+def test_parse_pytest_outcomes_strips_xfail_xpass_reasons() -> None:
+    output = (
+        "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked\n"
+        "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed\n"
+        "FAILED tests/test_mod.py::test_param[a - b] - assert left - right\n"
+        "PASSED tests/test_mod.py::test_ok"
+    )
+
+    assert vf.parse_pytest_outcomes(output) == {
+        "tests/test_mod.py::test_xfail": "XFAIL",
+        "tests/test_mod.py::test_xpass": "XPASS",
+        "tests/test_mod.py::test_param[a - b]": "FAILED",
+        "tests/test_mod.py::test_ok": "PASSED",
+    }
+
+
+def test_parse_judge_choice_uses_first_choice_after_verdict_marker() -> None:
+    response = "Final Judgment: B because it is a better answer"
+
+    assert vf.parse_judge_choice(response, choices=("A", "B")) == "B"

{verifiers-0.3.1.dev9 → verifiers-0.3.1.dev11}/verifiers/v1/__init__.py

@@ -26,7 +26,12 @@ from verifiers.v1.configs.serve import (
     StaticPoolConfig,
     pool_serve_kwargs,
 )
-from verifiers.v1.configs.task import TaskConfig
+from verifiers.v1.configs.task import (
+    DecoratedFunctionConfig,
+    FunctionConfig,
+    RewardFunctionConfig,
+    TaskConfig,
+)
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
@@ -240,6 +245,9 @@ __all__ = [  # noqa: RUF022 - grouped by public API area
     "Taskset",
     "TaskConfig",
     "TasksetConfig",
+    "FunctionConfig",
+    "DecoratedFunctionConfig",
+    "RewardFunctionConfig",
     "BaseConfig",
     "Harness",
     "HarnessSession",

verifiers-0.3.1.dev11/verifiers/v1/configs/task.py

@@ -0,0 +1,64 @@
+"""Run-time task knobs (`--env.taskset.task.*`), read by `Task` behavior."""
+
+from __future__ import annotations
+
+from pydantic import Field, FiniteFloat, model_validator
+from pydantic_config import BaseConfig
+
+from verifiers.v1.configs.judge import Judges, check_judges, resolve_judges
+
+
+class FunctionConfig(BaseConfig):
+    """A function plugged in by import path."""
+
+    fn: str
+    """Import path to the function: `pkg.module.function`, `pkg.module:function`,
+    or `path/to/file.py:function`."""
+
+
+class DecoratedFunctionConfig(FunctionConfig):
+    """A plugged function standing in for a decorated task method."""
+
+    priority: int = 0
+    """Execution order, like the decorator's — higher runs first, ties break by name."""
+
+
+class RewardFunctionConfig(DecoratedFunctionConfig):
+    """A plugged function standing in for a `@vf.reward` task method."""
+
+    weight: FiniteFloat = 1.0
+
+
+class TaskConfig(BaseConfig):
+    """Run-time knobs read by `Task` behavior.
+
+    Subclass for server placement, judge, or scoring settings. Every field needs a
+    default because constructing a task without a config builds the declared config type.
+    Load-time dataset settings belong on `TasksetConfig` instead.
+    """
+
+    judges: Judges = Field(default_factory=list)
+    """Judge plugins run after task rewards, set through `--env.taskset.task.judges`."""
+
+    stops: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
+    """Stop conditions `(trace) -> bool` plugged by name, merged with the task's
+    `@vf.stop` methods (a plugged function replaces a decorated one with the same
+    name)."""
+    metrics: dict[str, DecoratedFunctionConfig] = Field(default_factory=dict)
+    """Metrics `(task, trace, runtime) -> float` plugged by name, merged with the
+    task's `@vf.metric` methods."""
+    rewards: dict[str, RewardFunctionConfig] = Field(default_factory=dict)
+    """Weighted rewards `(task, trace, runtime) -> float` plugged by name, merged with
+    the task's `@vf.reward` methods."""
+
+    @model_validator(mode="before")
+    @classmethod
+    def _resolve_judges(cls, data):
+        if isinstance(data, dict) and data.get("judges"):
+            data["judges"] = resolve_judges(data["judges"])
+        return data
+
+    @model_validator(mode="after")
+    def _check_judges(self) -> TaskConfig:
+        check_judges(self.judges)
+        return self

{verifiers-0.3.1.dev9 → verifiers-0.3.1.dev11}/verifiers/v1/harnesses/openclaw/harness.py

@@ -117,8 +117,7 @@ class OpenClawHarness(Harness[OpenClawHarnessConfig]):
         data: TaskData,
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(data)
-        # Opt-in via sampling: OpenClaw redacts persisted encrypted reasoning, so resumed sessions 400 replaying it.
-        reasoning = ctx.sampling.reasoning_effort not in (None, "none")
+        provider, _ = ctx.model.split("/", 1)
         directory = OPENCLAW_DIR.format(version=self.config.version)
         state_dir = f".vf-openclaw/{trace.id}"
         config_path = f"{state_dir}/openclaw.json"
@@ -130,7 +129,7 @@ class OpenClawHarness(Harness[OpenClawHarnessConfig]):
                     "workspace": ".",
                     "skipBootstrap": True,
                     "sandbox": {"mode": "off"},
-                    "model": {"primary": f"intercept/{ctx.model}"},
+                    "model": {"primary": ctx.model},
                 }
             },
             "tools": {
@@ -140,21 +139,10 @@ class OpenClawHarness(Harness[OpenClawHarnessConfig]):
                 "deny": self.config.disabled_tools or [],
             },
             "models": {
-                "mode": "replace",
                 "providers": {
-                    "intercept": {
+                    provider: {
                         "baseUrl": endpoint,
                         "apiKey": "${OPENCLAW_INTERCEPT_KEY}",
-                        "api": "openai-responses",
-                        "authHeader": True,
-                        "models": [
-                            {
-                                "id": ctx.model,
-                                "name": ctx.model,
-                                "reasoning": reasoning,
-                                "input": ["text", "image"],
-                            }
-                        ],
                     }
                 },
             },

{verifiers-0.3.1.dev9 → verifiers-0.3.1.dev11}/verifiers/v1/harnesses/pi/harness.py

@@ -17,7 +17,6 @@ from verifiers.v1.trace import Trace
 
 logger = logging.getLogger(__name__)
 
-PROVIDER = "intercept"
 KEY_VAR = "PI_INTERCEPT_KEY"
 HOME_VAR = "VF_PI_ORIGINAL_HOME"
 
@@ -143,23 +142,12 @@ class PiHarness(Harness[PiHarnessConfig]):
     ) -> ProgramResult:
         system_prompt, prompt = self.resolve_prompt(data)
         agent_dir = f".vf-pi-agent-{trace.id}"
-        reasoning = ctx.sampling.reasoning_effort not in (
-            None,
-            "none",
-        ) or ctx.model.rsplit("/", 1)[-1].startswith(("gpt-5", "o1", "o3", "o4"))
+        provider, model = ctx.model.split("/", 1)
         models = {
             "providers": {
-                PROVIDER: {
+                provider: {
                     "baseUrl": endpoint,
-                    "api": "openai-completions",
                     "apiKey": f"${KEY_VAR}",
-                    "models": [
-                        {
-                            "id": ctx.model,
-                            "reasoning": reasoning,
-                            "input": ["text", "image"],
-                        }
-                    ],
                 }
             }
         }
@@ -202,9 +190,9 @@ class PiHarness(Harness[PiHarnessConfig]):
             PI_BIN,
             "--no-approve",
             "--provider",
-            PROVIDER,
+            provider,
             "--model",
-            ctx.model,
+            model,
             *mcp_args,
             *skill_args,
         ]

{verifiers-0.3.1.dev9 → verifiers-0.3.1.dev11}/verifiers/v1/rollout.py

@@ -30,7 +30,7 @@ from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import AgentInfo, Trace, TraceTask
 from verifiers.v1.types import Messages
-from verifiers.v1.utils.decorators import discover_decorated, invoke
+from verifiers.v1.utils.decorators import invoke
 
 logger = logging.getLogger(__name__)
 
@@ -91,9 +91,7 @@ class Rollout:
         )
         if on_trace is not None:
             on_trace(self.trace)
-        self._session = RolloutSession(
-            ctx, self.trace, discover_decorated(task, "stop"), limits
-        )
+        self._session = RolloutSession(ctx, self.trace, task.hooks("stop"), limits)
         self._stack = AsyncExitStack()
         self._failed = False
         self._failure: Exception | None = None

{verifiers-0.3.1.dev9 → verifiers-0.3.1.dev11}/verifiers/v1/task.py

@@ -12,8 +12,8 @@ from __future__ import annotations
 import copy
 import inspect
 import logging
-from collections.abc import Mapping
-from typing import TYPE_CHECKING, ClassVar, Generic, Self
+from collections.abc import Callable, Mapping
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, Self
 
 from pydantic import BaseModel, ConfigDict, Field
 from typing_extensions import TypeVar
@@ -164,8 +164,8 @@ class Task(Generic[DataT, StateT, ConfigT]):
             available["runtime"] = runtime
 
         async with boundary(TaskError, f"task {type(self).__name__} scoring"):
-            metrics = discover_decorated(self, "metric")
-            rewards = discover_decorated(self, "reward")
+            metrics = self.hooks("metric")
+            rewards = self.hooks("reward")
             if runtime is None:
                 skipped = [
                     fn.__name__ for fn in (*metrics, *rewards) if requires_runtime(fn)
@@ -231,6 +231,25 @@ class Task(Generic[DataT, StateT, ConfigT]):
 
         return [load_judge(config) for config in self.config.judges]
 
+    def hooks(self, attr: str) -> list[Callable[..., Any]]:
+        """The task's `@vf.<attr>` methods merged with the config-plugged functions —
+        a plugged function replaces a decorated method with the same name — sorted by
+        descending priority then name."""
+        from verifiers.v1.utils.loaders import load_plugged_fn
+
+        plugged = {
+            "stop": self.config.stops,
+            "metric": self.config.metrics,
+            "reward": self.config.rewards,
+        }[attr]
+        merged = {fn.__name__: fn for fn in discover_decorated(self, attr)}
+        merged |= {
+            name: load_plugged_fn(name, spec, attr) for name, spec in plugged.items()
+        }
+        fns = list(merged.values())
+        fns.sort(key=lambda fn: (-getattr(fn, f"{attr}_priority", 0), fn.__name__))
+        return fns
+
     @classmethod
     def toolsets(cls, config: ConfigT) -> list[Toolset]:
         """Tool servers launched per rollout, each constructed with its config off

{verifiers-0.3.1.dev9 → verifiers-0.3.1.dev11}/verifiers/v1/utils/loaders.py

@@ -1,11 +1,14 @@
-"""Resolve taskset, harness, judge, and environment plugins."""
+"""Resolve taskset, harness, judge, hook, and environment plugins."""
 
 import contextvars
+import functools
 import importlib
 import importlib.util
 import pkgutil
 from collections.abc import Callable
+from pathlib import Path
 from types import ModuleType
+from typing import Any
 
 from pydantic import ValidationError
 from pydantic_config import BaseConfig
@@ -13,6 +16,7 @@ from pydantic_config import BaseConfig
 from verifiers.v1.configs.env import EnvConfig
 from verifiers.v1.configs.harness import HarnessConfig
 from verifiers.v1.configs.judge import JudgeConfig
+from verifiers.v1.configs.task import DecoratedFunctionConfig, RewardFunctionConfig
 from verifiers.v1.configs.taskset import TasksetConfig
 from verifiers.v1.env import Env
 from verifiers.v1.envs.single_agent import SingleAgentEnv
@@ -205,6 +209,59 @@ def load_judge(config: JudgeConfig) -> Judge:
     return judge_class(config.id)(config)
 
 
+@functools.cache
+def file_module(path: str) -> ModuleType:
+    """Import a standalone `.py` file, cached by path so repeated plugged-fn loads
+    share one module object."""
+    resolved = Path(path).resolve()
+    spec = importlib.util.spec_from_file_location(resolved.stem, resolved)
+    if spec is None or spec.loader is None:
+        raise ImportError(f"cannot import module from {path!r}")
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def plugged_fn(path: str) -> Callable[..., Any]:
+    """Resolve a plugged function's import path: `pkg.module.function`,
+    `pkg.module:function`, or `path/to/file.py:function`."""
+    module_path, _, attr = path.rpartition(":")
+    if not module_path:
+        module_path, _, attr = path.rpartition(".")
+    if not module_path or not attr:
+        raise ValueError(
+            f"fn {path!r} must be `pkg.module.function`, `pkg.module:function`, "
+            "or `path/to/file.py:function`"
+        )
+    if module_path.endswith(".py"):
+        module = file_module(module_path)
+    else:
+        module = importlib.import_module(module_path)
+    fn = getattr(module, attr, None)
+    if not callable(fn):
+        raise TypeError(f"fn {path!r} is not a function (got {fn!r})")
+    return fn
+
+
+def load_plugged_fn(
+    name: str, config: DecoratedFunctionConfig, attr: str
+) -> Callable[..., Any]:
+    """Build a task hook from a config entry: the imported async function wrapped
+    under the plugged `name`, tagged like its `@vf.<attr>` equivalent."""
+    fn = plugged_fn(config.fn)
+
+    @functools.wraps(fn)
+    def plugged(*args: Any, **kwargs: Any) -> Any:
+        return fn(*args, **kwargs)
+
+    plugged.__name__ = name
+    setattr(plugged, attr, True)
+    setattr(plugged, f"{attr}_priority", config.priority)
+    if isinstance(config, RewardFunctionConfig):
+        plugged._vf_weight = config.weight
+    return plugged
+
+
 def taskset_config_type(taskset_id: str) -> type[TasksetConfig]:
     """Resolve the taskset's config specialization through its MRO."""
     return (

verifiers-0.3.1.dev9/tests/v1/test_scoring.py

@@ -1,31 +0,0 @@
-import verifiers.v1 as vf
-
-
-def test_compare_stdout_results_accepts_token_equal_text() -> None:
-    assert vf.compare_stdout_results("hello   world\n", "hello world\n")
-
-
-def test_compare_stdout_results_keeps_numeric_tolerance() -> None:
-    assert vf.compare_stdout_results("1.0001 2.0\n", "1.0002 2.0009\n")
-
-
-def test_parse_pytest_outcomes_strips_xfail_xpass_reasons() -> None:
-    output = (
-        "XFAIL tests/test_mod.py::test_xfail - known bug - still tracked\n"
-        "XPASS tests/test_mod.py::test_xpass - always xfail - unexpectedly passed\n"
-        "FAILED tests/test_mod.py::test_param[a - b] - assert left - right\n"
-        "PASSED tests/test_mod.py::test_ok"
-    )
-
-    assert vf.parse_pytest_outcomes(output) == {
-        "tests/test_mod.py::test_xfail": "XFAIL",
-        "tests/test_mod.py::test_xpass": "XPASS",
-        "tests/test_mod.py::test_param[a - b]": "FAILED",
-        "tests/test_mod.py::test_ok": "PASSED",
-    }
-
-
-def test_parse_judge_choice_uses_first_choice_after_verdict_marker() -> None:
-    response = "Final Judgment: B because it is a better answer"
-
-    assert vf.parse_judge_choice(response, choices=("A", "B")) == "B"

verifiers-0.3.1.dev9/verifiers/v1/configs/task.py

@@ -1,32 +0,0 @@
-"""Run-time task knobs (`--env.taskset.task.*`), read by `Task` behavior."""
-
-from __future__ import annotations
-
-from pydantic import Field, model_validator
-from pydantic_config import BaseConfig
-
-from verifiers.v1.configs.judge import Judges, check_judges, resolve_judges
-
-
-class TaskConfig(BaseConfig):
-    """Run-time knobs read by `Task` behavior.
-
-    Subclass for server placement, judge, or scoring settings. Every field needs a
-    default because constructing a task without a config builds the declared config type.
-    Load-time dataset settings belong on `TasksetConfig` instead.
-    """
-
-    judges: Judges = Field(default_factory=list)
-    """Judge plugins run after task rewards, set through `--env.taskset.task.judges`."""
-
-    @model_validator(mode="before")
-    @classmethod
-    def _resolve_judges(cls, data):
-        if isinstance(data, dict) and data.get("judges"):
-            data["judges"] = resolve_judges(data["judges"])
-        return data
-
-    @model_validator(mode="after")
-    def _check_judges(self) -> TaskConfig:
-        check_judges(self.judges)
-        return self