verifiers 0.4.1.dev58__tar.gz → 0.4.1.dev65__tar.gz

{verifiers-0.4.1.dev58 → verifiers-0.4.1.dev65}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev58
+Version: 0.4.1.dev65
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -36,11 +36,11 @@ Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
 Requires-Dist: prime-runs>=0.1.4
-Requires-Dist: prime-sandboxes>=0.4.7
+Requires-Dist: prime-sandboxes>=0.5.0
 Requires-Dist: prime-tunnel>=0.1.14
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
-Requires-Dist: renderers[multimodal]>=0.1.12
+Requires-Dist: renderers[multimodal]>=0.1.13.dev16
 Requires-Dist: rich>=11.0.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0

{verifiers-0.4.1.dev58 → verifiers-0.4.1.dev65}/pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "math-verify>=0.8.0",
     "openai>=2.54.0,<3.0.0",
     "prime-tunnel>=0.1.14",
-    "prime-sandboxes>=0.4.7",
+    "prime-sandboxes>=0.5.0",
     "prime-runs>=0.1.4",
     "pydantic>=2.12.3",
     "rich>=11.0.0",
@@ -49,7 +49,7 @@ dependencies = [
     "prime-pydantic-config[toml]>=0.4.3",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",
-    "renderers[multimodal]>=0.1.12",
+    "renderers[multimodal]>=0.1.13.dev16",
     "typing_extensions>=4.12.2",
     "mcp==2.0.0",
 ]

{verifiers-0.4.1.dev58 → verifiers-0.4.1.dev65}/tests/integration/fixtures/counter_tool.py

@@ -21,6 +21,8 @@ class CounterToolset(vf.Toolset[vf.ToolsetConfig, CounterState]):
 
 class CounterTaskConfig(vf.TaskConfig):
     tools: vf.ToolsetConfig = vf.ToolsetConfig()
+    constant_reward: bool = False
+    """Score every rollout 1, so a skipped score (reward 0) stands out."""
 
 
 class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
@@ -30,7 +32,7 @@ class CounterTask(vf.Task[vf.TaskData, CounterState, CounterTaskConfig]):
 
     @vf.reward(weight=1.0)
     async def counted(self, trace: vf.Trace) -> float:
-        return float(trace.state.count >= 2)
+        return float(self.config.constant_reward or trace.state.count >= 2)
 
 
 class CounterConfig(vf.TasksetConfig):

{verifiers-0.4.1.dev58 → verifiers-0.4.1.dev65}/tests/integration/test_limits.py

@@ -41,6 +41,44 @@ async def test_rollout_timeout(run):
     assert trace.stop_condition == "agent_timeout"
 
 
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_truncated(run, skip):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        taskset_overrides={"task": {"constant_reward": True}},
+        env={"agent": {"max_turns": 1, "scoring": {"skip_on_truncated": skip}}},
+    )
+    (trace,) = episode.traces
+    assert trace.is_truncated and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "truncated"
+    else:
+        assert trace.reward == 1
+        assert "scoring_skipped" not in trace.info
+
+
+@pytest.mark.parametrize("skip", [False, True], ids=["scored", "skipped"])
+async def test_skip_scoring_on_timeout(run, skip):
+    (episode,) = await run(
+        "counter-tool",
+        harness="null",
+        runtime={"type": "prime"},
+        rollout_timeout=1,
+        taskset_overrides={"task": {"constant_reward": True}},
+        env={"agent": {"scoring": {"skip_on_timeout": skip}}},
+    )
+    (trace,) = episode.traces
+    assert trace.is_timeout and trace.ok
+    if skip:
+        assert trace.reward == 0 and trace.rewards == {}
+        assert trace.info["scoring_skipped"] == "timeout"
+    else:
+        assert trace.reward == 1
+
+
 @pytest.mark.parametrize("harness,harness_runtime", grid(needs=("mcp",)), indirect=True)
 async def test_max_turns(run, harness, harness_runtime):
     (episode,) = await run(

verifiers-0.4.1.dev65/tests/unit/test_api.py

@@ -0,0 +1,66 @@
+"""The `verifiers.v1` namespace: its lazy names stay in sync with what type checkers
+see, and the config tree imports without the runtime."""
+
+import ast
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import verifiers.v1 as vf
+
+# What the config tree may load: a trainer validates its configs against it in an
+# environment that installs nothing else.
+CONFIG_TREE_PACKAGES = {
+    "annotated_types",
+    "pydantic",
+    "pydantic_config",
+    "pydantic_core",
+    "renderers",
+    "tomli",
+    "tomli_w",
+    "typing_extensions",
+    "typing_inspection",
+    "verifiers",
+}
+
+IMPORT_CONFIG_TREE = """
+import importlib, json, pkgutil, sys
+import verifiers.v1.configs as configs
+for module in pkgutil.walk_packages(configs.__path__, "verifiers.v1.configs."):
+    importlib.import_module(module.name)
+packages = {name.partition(".")[0] for name in sys.modules if not name.startswith("_")}
+print(json.dumps(sorted(packages - sys.stdlib_module_names)))
+"""
+
+
+def type_checking_name_to_module() -> dict[str, str]:
+    tree = ast.parse(Path(vf.__file__).read_text())
+    block = next(node for node in tree.body if isinstance(node, ast.If))
+    return {alias.name: node.module for node in block.body for alias in node.names}
+
+
+def test_lazy_names_match_type_checking_imports():
+    assert vf._NAME_TO_MODULE == type_checking_name_to_module()
+
+
+def test_all_exports_every_lazy_name():
+    assert sorted(vf.__all__) == sorted(vf._NAME_TO_MODULE)
+
+
+def test_every_lazy_name_resolves():
+    resolved = {name: getattr(vf, name) for name in vf.__all__}
+
+    assert sorted(resolved) == sorted(vf.__all__)
+
+
+def test_config_tree_imports_without_the_runtime():
+    result = subprocess.run(
+        [sys.executable, "-c", IMPORT_CONFIG_TREE],
+        capture_output=True,
+        check=True,
+        text=True,
+        timeout=60,
+    )
+
+    assert set(json.loads(result.stdout)) - CONFIG_TREE_PACKAGES == set()

{verifiers-0.4.1.dev58 → verifiers-0.4.1.dev65}/tests/unit/test_trace.py

@@ -9,6 +9,7 @@ from verifiers.v1.configs.agent import resolve_agent
 from verifiers.v1.dialects.chat import ChatDialect
 from verifiers.v1.dialects.responses import ResponsesDialect, fold_assistant
 from verifiers.v1.graph import MessageNode, prepare_turn
+from verifiers.v1.session import RolloutLimits
 from verifiers.v1.types import AssistantMessage, UserMessage
 from verifiers.v1.utils.trace_store import write_episode
 
@@ -266,3 +267,29 @@ def test_sampling_mask_from_packed_matches_lists():
     ):
         np.testing.assert_array_equal(mask.ids, ids)
         np.testing.assert_array_equal(mask.counts, counts)
+
+
+def test_max_turn_tokens_limit():
+    limits = RolloutLimits(max_turn_tokens=16)
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt=None)),
+    )
+    trace.calls.append(vf.ModelCall(finish_reason="stop"))
+    assert limits.reached(trace) is None
+    # A length finish below the cap ran out of context: not this limit.
+    usage = vf.Usage(prompt_tokens=100, completion_tokens=8)
+    trace.calls.append(vf.ModelCall(finish_reason="length", usage=usage))
+    assert limits.reached(trace) is None
+    usage = vf.Usage(prompt_tokens=100, completion_tokens=16)
+    trace.calls.append(vf.ModelCall(finish_reason="length", usage=usage))
+    assert limits.reached(trace) == "max_turn_tokens"
+    trace.stop("max_turn_tokens")
+    assert trace.is_truncated
+
+
+@pytest.mark.parametrize("max_tokens,expected", [(None, 16), (8, 8), (64, 16)])
+def test_max_turn_tokens_caps_sampling(max_tokens, expected):
+    spec = vf.AgentConfig(max_turn_tokens=16)
+    sampling = vf.SamplingConfig(max_tokens=max_tokens)
+    assert resolve_agent(spec, sampling=sampling).sampling.max_tokens == expected