verifiers 0.1.15.dev443__tar.gz → 0.2.1__tar.gz

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev443
+Version: 0.2.1
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -24,56 +24,42 @@ Requires-Python: <3.14,>=3.11
 Requires-Dist: aiohttp>=3.9.0
 Requires-Dist: aiolimiter>=1.2.1
 Requires-Dist: anthropic>=0.78.0
-Requires-Dist: datasets<4.7.0,>=3.0.0
-Requires-Dist: gepa
+Requires-Dist: datasets<6.0.0,>=3.3.0
+Requires-Dist: gepa>=0.0.6
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: loguru>=0.7.0
 Requires-Dist: math-verify>=0.8.0
-Requires-Dist: mcp>=1.14.1
+Requires-Dist: mcp<2,>=1.24.0
 Requires-Dist: msgpack>=1.1.2
-Requires-Dist: numpy
-Requires-Dist: openai-agents>=0.0.7
-Requires-Dist: openai>=1.108.1
-Requires-Dist: prime-pydantic-config[toml]>=0.3.0.dev86
-Requires-Dist: prime-sandboxes>=0.2.27
+Requires-Dist: numpy>=2.1.0
+Requires-Dist: openai-agents>=0.8.2
+Requires-Dist: openai>=2.9.0
+Requires-Dist: prime-pydantic-config[toml]>=0.4.2
+Requires-Dist: prime-sandboxes>=0.2.31
 Requires-Dist: prime-tunnel>=0.1.8
-Requires-Dist: pydantic>=2.11.9
+Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0
-Requires-Dist: renderers>=0.1.8.dev40
+Requires-Dist: renderers>=0.1.8
 Requires-Dist: requests
-Requires-Dist: rich
+Requires-Dist: rich>=11.0.0
 Requires-Dist: setproctitle>=1.3.0
 Requires-Dist: tenacity>=8.5.0
 Requires-Dist: tomli-w>=1.0.0
-Requires-Dist: typing-extensions; python_version < '3.12'
+Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'
 Provides-Extra: browser
-Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
-Requires-Dist: stagehand>=3.0.0; extra == 'browser'
+Requires-Dist: stagehand>=3.4.0; extra == 'browser'
 Provides-Extra: harbor
 Requires-Dist: harbor==0.14.0; (python_version >= '3.12') and extra == 'harbor'
 Provides-Extra: modal
 Requires-Dist: modal>=1.4.0; extra == 'modal'
 Provides-Extra: notebook
 Requires-Dist: nest-asyncio>=1.6.0; extra == 'notebook'
-Provides-Extra: quest
-Requires-Dist: certifi; extra == 'quest'
-Requires-Dist: pillow; extra == 'quest'
-Requires-Dist: pymupdf; extra == 'quest'
+Provides-Extra: openenv
+Requires-Dist: openenv>=0.4.1; extra == 'openenv'
 Provides-Extra: rg
-Requires-Dist: reasoning-gym; extra == 'rg'
-Provides-Extra: rl
-Requires-Dist: accelerate>=1.4.0; extra == 'rl'
-Requires-Dist: deepspeed>=0.17.6; extra == 'rl'
-Requires-Dist: flash-attn>=2.8.3; extra == 'rl'
-Requires-Dist: liger-kernel>=0.5.10; extra == 'rl'
-Requires-Dist: peft; extra == 'rl'
-Requires-Dist: requests; extra == 'rl'
-Requires-Dist: torch<2.9.0,>=2.8.0; extra == 'rl'
-Requires-Dist: transformers>=4.56.2; extra == 'rl'
-Requires-Dist: vllm<0.11.0,>=0.10.0; extra == 'rl'
-Requires-Dist: wandb; extra == 'rl'
+Requires-Dist: reasoning-gym>=0.1.8; extra == 'rg'
 Provides-Extra: ta
 Requires-Dist: nltk>=3.9.2; extra == 'ta'
 Requires-Dist: textarena==0.7.4; extra == 'ta'

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/pyproject.toml

@@ -28,72 +28,69 @@ classifiers = [
 
 dependencies = [
     "anthropic>=0.78.0",
-    "datasets>=3.0.0, <4.7.0",
-    "numpy",
+    "datasets>=3.3.0,<6.0.0",
+    "numpy>=2.1.0",
     "math-verify>=0.8.0",
-    "mcp>=1.14.1",
-    "openai>=1.108.1",
-    "openai-agents>=0.0.7",
+    "mcp>=1.24.0,<2",
+    "openai>=2.9.0",
+    "openai-agents>=0.8.2",
     "prime-tunnel>=0.1.8",
-    "prime-sandboxes>=0.2.27",
-    "pydantic>=2.11.9",
+    "prime-sandboxes>=0.2.31",
+    "pydantic>=2.12.3",
     "requests",
-    "rich",
+    "rich>=11.0.0",
     "tenacity>=8.5.0",
-    "gepa",
+    "gepa>=0.0.6",
     "pyzmq>=27.1.0",
     "msgpack>=1.1.2",
     "aiolimiter>=1.2.1",
     "setproctitle>=1.3.0",
     "httpx>=0.27.0",
     "aiohttp>=3.9.0",
-    "prime-pydantic-config[toml]>=0.3.0.dev86",
+    "prime-pydantic-config[toml]>=0.4.2",
     "uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",
-    "renderers>=0.1.8.dev40",
-    "typing_extensions; python_version < '3.12'",
+    "renderers>=0.1.8",
+    "typing_extensions>=4.12.2",
 ]
 
 [dependency-groups]
 dev = [
-    "ruff",
-    "pre-commit",
+    "ruff>=0.9.0",
+    "pre-commit>=2.19.0",
     "ty>=0.0.1a29,<0.0.22",
-    "pytest>=7.0.0",
+    "pytest>=7.0.0,<9.1.0",
     "pytest-asyncio>=0.21.0",
     "pytest-cov>=4.0.0",
     "pytest-xdist>=3.8.0",
-    "ipykernel",
-    "ipywidgets",
-    "reasoning-gym",
-    "stagehand>=3.0.0",
-    "aiohttp>=3.9.0",
+    "nest-asyncio>=1.6.0",
+    "reasoning-gym>=0.1.8",
+    "stagehand>=3.4.0",
     "python-dotenv>=1.0.0",
     "nltk>=3.9.2",
     "textarena==0.7.4",
 ]
-# v1 example environments vendored under environments/, resolved by id via [tool.uv.sources].
+# V1 examples and support packages, resolved from their local sources below.
 # Installed by default (tool.uv.default-groups) so the v1 CLIs (eval <id>, serve) run straight
 # from this repo. The built-in tasksets/harnesses ship inside verifiers (verifiers/v1/...).
 examples = [
     "compact",
     "gsm8k-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
     "reverse-text-v1", "color-codeword-v1",
-    "wordle-v1", "alphabet-sort-v1", "scratchpad-v1",
+    "wordle-v1", "openenv-wordle-v1", "alphabet-sort-v1", "scratchpad-v1",
 ]
 
 [project.optional-dependencies]
 rg = [
-    "reasoning-gym",
+    "reasoning-gym>=0.1.8",
 ]
 ta = [
     "textarena==0.7.4",
     "nltk>=3.9.2",
 ]
 browser = [
-    "stagehand>=3.0.0",
-    "aiohttp>=3.9.0",
+    "stagehand>=3.4.0",
     "python-dotenv>=1.0.0",
 ]
 harbor = [
@@ -106,24 +103,9 @@ modal = [
 notebook = [
     "nest-asyncio>=1.6.0",
 ]
-quest = [
-    "certifi",
-    "pillow",
-    "pymupdf",
-]
-rl = [
-    "torch>=2.8.0,<2.9.0",
-    "transformers>=4.56.2",
-    "accelerate>=1.4.0",
-    "requests",
-    "peft",
-    "wandb",
-    "vllm>=0.10.0,<0.11.0",
-    "liger-kernel>=0.5.10",
-    "deepspeed>=0.17.6",
-    "flash-attn>=2.8.3",
+openenv = [
+    "openenv>=0.4.1",
 ]
-
 [tool.uv]
 preview = true
 required-version = ">=0.11.1"
@@ -143,8 +125,8 @@ wiki-search-v1 = { path = "environments/wiki_search_v1", editable = true }
 gsm8k-v1 = { path = "environments/gsm8k_v1", editable = true }
 code-golf-v1 = { path = "environments/code_golf_v1", editable = true }
 reverse-text-v1 = { path = "environments/reverse_text_v1", editable = true }
-reverse-text = { path = "environments/reverse_text", editable = true }
 wordle-v1 = { path = "environments/wordle_v1", editable = true }
+openenv-wordle-v1 = { path = "environments/openenv_wordle_v1", editable = true }
 color-codeword-v1 = { path = "environments/color_codeword_v1", editable = true }
 alphabet-sort-v1 = { path = "environments/alphabet_sort_v1", editable = true }
 scratchpad-v1 = { path = "environments/scratchpad_v1", editable = true }
@@ -156,12 +138,6 @@ prime-sandboxes = false
 prime-pydantic-config = false
 renderers = false
 
-[tool.uv.extra-build-dependencies]
-flash-attn = [{ requirement = "torch", match-runtime = true }]
-
-[tool.uv.extra-build-variables]
-flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE" }
-
 [project.scripts]
 vf-eval = "verifiers.scripts.eval:main"
 eval = "verifiers.v1.cli.eval.main:main"
@@ -170,6 +146,7 @@ debug = "verifiers.v1.cli.debug:main"
 replay = "verifiers.v1.cli.replay:main"
 serve = "verifiers.v1.cli.serve:main"
 init = "verifiers.v1.cli.init:main"
+gepa = "verifiers.v1.cli.gepa:main"
 vf-gepa = "verifiers.scripts.gepa:main"
 vf-init = "verifiers.scripts.init:main"
 vf-install = "verifiers.scripts.install:main"
@@ -248,7 +225,7 @@ markers = [
     "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
     "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
     "null: v1 e2e cases on the null harness",
-    "default: v1 e2e cases on the default harness",
+    "bash: v1 e2e cases on the bash harness",
     "rlm: v1 e2e cases on the rlm harness",
     "kimi_code: v1 e2e cases on the kimi-code harness",
     "codex: v1 e2e cases on the codex harness",

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/conftest.py

@@ -39,6 +39,12 @@ from verifiers.types import (
 from verifiers.utils.save_utils import state_to_output
 
 
+def pytest_configure(config) -> None:
+    config.addinivalue_line(
+        "markers", "claude_code: v1 e2e cases on the claude-code harness"
+    )
+
+
 @pytest.fixture
 def basic_parser():
     """Return a basic Parser instance."""

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/test_browser_env.py

@@ -452,98 +452,6 @@ class TestDOMModeLLMConfig:
         assert config is None
 
 
-# ============================================================================
-# Example Environment Tests
-# ============================================================================
-
-
-class TestExampleDatasets:
-    """Tests for example environment datasets."""
-
-    def test_dom_example_dataset_structure(self):
-        """Test DOM example dataset has correct structure."""
-        from environments.browser_dom_example.browser_dom_example import (
-            create_example_dataset,
-        )
-
-        dataset = create_example_dataset()
-
-        assert "question" in dataset.column_names
-        assert "answer" in dataset.column_names
-        assert "start_url" in dataset.column_names
-        assert "task_id" in dataset.column_names
-        assert len(dataset) >= 1
-
-    def test_cua_example_dataset_structure(self):
-        """Test CUA example dataset has correct structure."""
-        from environments.browser_cua_example.browser_cua_example import (
-            create_example_dataset,
-        )
-
-        dataset = create_example_dataset()
-
-        assert "question" in dataset.column_names
-        assert "answer" in dataset.column_names
-        assert "start_url" in dataset.column_names
-        assert "task_id" in dataset.column_names
-        assert len(dataset) >= 1
-
-
-class TestJudgeAnswer:
-    """Tests for judge_answer reward function."""
-
-    @pytest.mark.asyncio
-    async def test_judge_answer_returns_1_for_yes(self):
-        """Test that judge_answer returns 1.0 when judge says yes."""
-        from environments.browser_dom_example.browser_dom_example import judge_answer
-
-        async def mock_judge(prompt, completion, answer, state):
-            return "yes, the answer is correct"
-
-        result = await judge_answer(
-            judge=mock_judge,
-            prompt="What is 2+2?",
-            completion=[{"role": "assistant", "content": "The answer is 4"}],
-            answer="4",
-            state={},
-        )
-        assert result == 1.0
-
-    @pytest.mark.asyncio
-    async def test_judge_answer_returns_0_for_no(self):
-        """Test that judge_answer returns 0.0 when judge says no."""
-        from environments.browser_dom_example.browser_dom_example import judge_answer
-
-        async def mock_judge(prompt, completion, answer, state):
-            return "no, the answer is incorrect"
-
-        result = await judge_answer(
-            judge=mock_judge,
-            prompt="What is 2+2?",
-            completion=[{"role": "assistant", "content": "The answer is 5"}],
-            answer="4",
-            state={},
-        )
-        assert result == 0.0
-
-    @pytest.mark.asyncio
-    async def test_judge_answer_case_insensitive(self):
-        """Test that judge response check is case insensitive."""
-        from environments.browser_dom_example.browser_dom_example import judge_answer
-
-        async def mock_judge(prompt, completion, answer, state):
-            return "YES"
-
-        result = await judge_answer(
-            judge=mock_judge,
-            prompt="test",
-            completion="test",
-            answer="test",
-            state={},
-        )
-        assert result == 1.0
-
-
 # ============================================================================
 # Constants Tests
 # ============================================================================

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/test_install_utils.py

@@ -73,7 +73,7 @@ class TestIsHubEnv:
         assert is_hub_env("gsm8k") is False
 
     def test_local_env_relative_path(self):
-        assert is_hub_env("./environments/gsm8k") is False
+        assert is_hub_env("./environments/gsm8k_v1") is False
 
     def test_local_env_absolute_path(self):
         assert is_hub_env("/path/to/gsm8k") is False

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/v1/conftest.py

@@ -15,15 +15,16 @@ Every matrix value carries a pytest mark, so subsets select with `-m`:
     uv run pytest tests/v1 -n auto                                # the whole matrix (needs modal setup)
     uv run pytest tests/v1 -n auto -m "not prime and not modal"  # the CI matrix (host + docker only)
     uv run pytest tests/v1 -n auto -m docker                      # any case touching the docker runtime
-    uv run pytest tests/v1 -n auto -m default                     # only the default harness
+    uv run pytest tests/v1 -n auto -m bash                        # only the bash harness
     uv run pytest tests/v1 -n auto -m prime                       # only prime (real sandboxes; local)
     uv run pytest tests/v1 -n auto -m modal                       # only modal (needs local setup)
 
 Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
-harnesses `null` / `default` / `rlm` / `kimi_code` / `codex`. A mark is applied per axis, so it
-selects every case touching that value on ANY axis; for one exact combination use `-k` on the test
-id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real remote
-sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs `-m "not prime and not modal"`.
+harnesses `null` / `bash` / `rlm` / `kimi_code` / `codex` / `claude_code`. A mark is applied per
+axis, so it selects every case touching that value on ANY axis; for one exact combination use `-k`
+on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`). prime/modal provision real
+remote sandboxes (slow, infra-flaky, need setup), so they're local-only — CI runs
+`-m "not prime and not modal"`.
 """
 
 import os
@@ -103,18 +104,19 @@ def tool_runtime(request) -> dict:
     return {"runtime": {"type": request.param}}
 
 
-# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m default`, ...).
-# Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex`)
-# install their dependencies at rollout. `compact` (an example harness) and `terminus-2` (drives
-# the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no shell);
-# `test_single_turn` skips `codex` (a coding agent, unreliable on a no-op echo).
+# Harnesses, composed with the runtime fixtures, each carrying its harness mark (`-m bash`, ...).
+# Built-ins are bundled in the `harnesses` package; the agent CLIs (`rlm` / `kimi-code` / `codex` /
+# `claude-code`) install their dependencies at rollout. `compact` (an example harness) and
+# `terminus-2` (drives the host tmux) are excluded. `test_agentic` skips `null` (a chat loop with no
+# shell); `test_single_turn` skips the coding agents, which are unreliable on a no-op echo.
 @pytest.fixture(
     params=[
         pytest.param("null", marks=pytest.mark.null, id="null"),
-        pytest.param("default", marks=pytest.mark.default, id="default"),
+        pytest.param("bash", marks=pytest.mark.bash, id="bash"),
         pytest.param("rlm", marks=pytest.mark.rlm, id="rlm"),
         pytest.param("kimi-code", marks=pytest.mark.kimi_code, id="kimi-code"),
         pytest.param("codex", marks=pytest.mark.codex, id="codex"),
+        pytest.param("claude-code", marks=pytest.mark.claude_code, id="claude-code"),
     ]
 )
 def harness(request) -> str:
@@ -172,6 +174,7 @@ def _eval_config(
     harness_overrides: dict | None = None,
     pool: dict | None = None,
     model: str | None = None,
+    reasoning_effort: str | None = None,
 ) -> EvalConfig:
     """Build the smallest `EvalConfig` that still exercises the path, shared by the in-process
     (`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` / `harness_overrides`
@@ -192,7 +195,11 @@ def _eval_config(
         num_rollouts=n,
         max_turns=max_turns,
         max_output_tokens=max_tokens,
-        sampling={"max_tokens": max_tokens, "temperature": 0},
+        sampling={
+            "max_tokens": max_tokens,
+            "temperature": 0,
+            "reasoning_effort": reasoning_effort,
+        },
         timeout={"rollout": rollout_timeout, "scoring": 60},
         retries={"rollout": {"max_retries": 2, "include": ["ProviderError"]}},
         rich=False,

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/v1/fixtures/echo_user_sim_v1.py

@@ -35,7 +35,7 @@ class EchoUserSimUser(vf.User[vf.UserConfig, EchoUserSimState]):
         if self.turns >= len(self.phrases):
             self.state.user_finished = True
             return []
-        return [{"role": "user", "content": self.phrases[self.turns]}]
+        return [vf.UserMessage(content=self.phrases[self.turns])]
 
 
 class EchoUserSimData(vf.TaskData):

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/v1/fixtures/tool_response_image_v1.py

@@ -7,7 +7,7 @@ content part on a ToolMessage.
 
 import verifiers.v1 as vf
 
-PNG_DATA = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADUlEQVR4nGP4z8DwHwAFAAH/e+m+7wAAAABJRU5ErkJggg=="
+PNG_DATA = "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAIAAAD8GO2jAAAAKUlEQVRIx+3NMQEAAAjDMMC/52ECvlRA06nUZ/N6BwAAAAAAAAAAAIcttVsCPm9Ue3AAAAAASUVORK5CYII="
 EXPECTED_URL = f"data:image/png;base64,{PNG_DATA}"
 SYSTEM = "Call the requested tool before answering."
 

verifiers-0.2.1/tests/v1/test_color_codeword_tasks.py

@@ -0,0 +1,18 @@
+from color_codeword_v1 import taskset
+
+
+def test_color_images_are_rendered_once(monkeypatch) -> None:
+    calls: list[tuple[int, int, int]] = []
+    render = taskset.image_data_url
+
+    def counted(image) -> str:
+        calls.append(image.getpixel((0, 0)))
+        return render(image)
+
+    monkeypatch.setattr(taskset, "image_data_url", counted)
+    tasks = taskset.ColorCodewordTaskset(taskset.ColorCodewordConfig()).select(
+        num_tasks=100
+    )
+
+    assert len(tasks) == 100
+    assert calls == list(taskset.COLOR_RGB.values())

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/v1/test_e2e.py

@@ -6,8 +6,8 @@ import pytest
 @pytest.mark.e2e
 async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     """Single-turn (echo a short phrase back)."""
-    if harness == "codex":
-        pytest.skip("codex is a coding agent, not reliable on a no-op echo chat task")
+    if harness in {"codex", "claude-code"}:
+        pytest.skip("coding agents are not reliable on a no-op echo chat task")
     (trace,) = await run_v1(
         "echo-v1",
         harness=harness,
@@ -18,6 +18,12 @@ async def test_single_turn(run_v1, harness, harness_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns == 1
     assert trace.reward == 1.0
+    # Every sampled turn has one per-call record, linked to its assistant node.
+    sampled = [i for i, n in enumerate(trace.nodes) if n.sampled]
+    assert [c.node for c in trace.calls if c.error is None] == sampled
+    for call in trace.calls:
+        assert call.model and call.sampling is not None
+        assert call.time.duration > 0
 
 
 @pytest.mark.e2e
@@ -74,6 +80,11 @@ async def test_tool(run_v1, harness_runtime, tool_runtime, tmp_path):
     assert trace.errors == []
     assert trace.num_turns >= 2  # tool call + answer
     assert trace.reward == 1.0
+    # The interception server captured the advertised tools onto the trace (for tool-use SFT):
+    # the null harness offered the task's MCP tool as `echo_back`, schema included.
+    assert trace.tools is not None
+    (echo_tool,) = [t for t in trace.tools if t.name == "echo_back"]
+    assert "message" in echo_tool.parameters.get("properties", {})
 
 
 @pytest.mark.e2e
@@ -140,7 +151,8 @@ async def test_tool_response_image(run_v1, tmp_path):
         "tool-response-image-v1",
         harness="null",
         harness_overrides={"runtime": {"type": "subprocess"}},
-        model="qwen/qwen3-vl-8b-instruct",
+        model="openai/gpt-5.6-luna",
+        reasoning_effort="none",
         output_dir=tmp_path,
         max_turns=4,
     )

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/v1/test_envs.py

@@ -1,10 +1,8 @@
-"""Smoke-eval every v1 example taskset in `environments/` through the `eval` CLI.
+"""Smoke-eval every V1 example taskset in `environments/` through the `eval` CLI.
 
-The v0 counterpart (`tests/test_envs.py`) covers v0 envs (`vf.load_environment` + `vf-eval`);
-here we run each `_v1` taskset with its required harness for one short, capped rollout and
-require it to succeed — so a broken example taskset fails CI. `compact` is excluded (it's a
-harness, not a taskset); SWE/container tasksets need a docker/prime runtime and are covered by
-the dedicated v1 e2e tests instead.
+Each `_v1` taskset runs with its required harness for one short, capped rollout, so a broken
+example taskset fails CI. `compact` is excluded (it's a harness, not a taskset);
+SWE/container tasksets need a docker/prime runtime and are covered by dedicated V1 e2e tests.
 """
 
 import os

verifiers-0.2.1/tests/v1/test_taskset.py

@@ -0,0 +1,71 @@
+"""`Taskset.select` over list, generator, and `INFINITE` `load` implementations."""
+
+import itertools
+
+import pytest
+
+import verifiers.v1 as vf
+
+
+class CountTask(vf.Task[vf.TaskData]):
+    pass
+
+
+class InfiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    INFINITE = True
+
+    def load(self):
+        for i in itertools.count():
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+class FiniteTaskset(vf.Taskset[CountTask, vf.TasksetConfig]):
+    def load(self):
+        for i in range(10):
+            yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+
+def idxs(tasks: list[CountTask]) -> list[int]:
+    return [task.data.idx for task in tasks]
+
+
+def test_select_bounds_an_infinite_taskset() -> None:
+    tasks = InfiniteTaskset(vf.TasksetConfig()).select(num_tasks=5)
+    assert idxs(tasks) == [0, 1, 2, 3, 4]
+
+
+def test_select_infinite_requires_num_tasks() -> None:
+    with pytest.raises(ValueError, match="infinite"):
+        InfiniteTaskset(vf.TasksetConfig()).select()
+
+
+def test_select_infinite_ignores_shuffle() -> None:
+    tasks = InfiniteTaskset(vf.TasksetConfig()).select(num_tasks=3, shuffle=True)
+    assert idxs(tasks) == [0, 1, 2]
+
+
+def test_select_finite() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    assert len(taskset.select()) == 10
+    assert idxs(taskset.select(num_tasks=4)) == [0, 1, 2, 3]
+
+
+def test_select_shuffle_samples_the_whole_taskset_reproducibly() -> None:
+    taskset = FiniteTaskset(vf.TasksetConfig())
+    first = idxs(taskset.select(num_tasks=5, shuffle=True))
+    assert first == idxs(taskset.select(num_tasks=5, shuffle=True))
+    assert len(first) == 5 and set(first) <= set(range(10))
+    assert first != [0, 1, 2, 3, 4]  # sampled from the whole set, not the head
+
+
+def test_select_only_builds_what_the_run_takes() -> None:
+    built: list[int] = []
+
+    class RecordingTaskset(InfiniteTaskset):
+        def load(self):
+            for i in itertools.count():
+                built.append(i)
+                yield CountTask(vf.TaskData(idx=i, prompt=f"task {i}"))
+
+    RecordingTaskset(vf.TasksetConfig()).select(num_tasks=3)
+    assert built == [0, 1, 2]

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/tests/v1/test_trace.py

@@ -60,6 +60,7 @@ def test_wire_trace_round_trip():
     # carry node `parent` links for `num_branches` to survive.
     tr = vf.Trace[MyTask, vf.State](
         task=vf.TraceTask(type="MyTask", data=MyTask(idx=0, prompt="q", answer="a")),
+        tools=[vf.Tool(name="echo", description="", parameters={"type": "object"})],
         nodes=[
             MessageNode(parent=None, message=UserMessage(content="q"), sampled=False),
             MessageNode(parent=0, message=AssistantMessage(content="a1"), sampled=True),
@@ -80,6 +81,9 @@ def test_wire_trace_round_trip():
     assert rt.reward == 1.0  # property recomputed from `rewards`
     assert rt.stop_condition == "done"
     assert rt.info == {"build": "ok"}
+    assert (
+        rt.tools == tr.tools
+    )  # the advertised tools persist (tool-use SFT reads them)
     assert rt.task.data.model_extra == {
         "answer": "a"
     }  # taskset extras preserved on WireTaskData

{verifiers-0.1.15.dev443 → verifiers-0.2.1}/verifiers/envs/integrations/browser_env/README.md

@@ -139,16 +139,3 @@ CUA execution options:
 1. **Prebuilt image** (default): fastest startup
 2. **Binary upload** (`use_prebuilt_image=False`): custom server workflows
 3. **Manual local server** (`use_sandbox=False`): local development/debugging
-
-## Example Environments
-
-For complete reference implementations, see:
-
-- **DOM example:** `environments/browser_dom_example/`
-  - `environments/browser_dom_example/browser_dom_example.py`
-  - `environments/browser_dom_example/README.md`
-- **CUA example:** `environments/browser_cua_example/`
-  - `environments/browser_cua_example/browser_cua_example.py`
-  - `environments/browser_cua_example/README.md`
-
-These examples show end-to-end `load_environment()` setup, evaluation commands, and recommended runtime flags.