verifiers 0.2.1.dev51__tar.gz → 0.2.1.dev53__tar.gz

{verifiers-0.2.1.dev51 → verifiers-0.2.1.dev53}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev51
+Version: 0.2.1.dev53
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev51 → verifiers-0.2.1.dev53}/pyproject.toml

@@ -80,7 +80,7 @@ examples = [
     "compact",
     "gsm8k-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
     "reverse-text-v1", "color-codeword-v1",
-    "wordle-v1", "alphabet-sort-v1", "scratchpad-v1", "mmmu-v1",
+    "wordle-v1", "alphabet-sort-v1", "scratchpad-v1",
 ]
 
 [project.optional-dependencies]
@@ -143,12 +143,10 @@ wiki-search-v1 = { path = "environments/wiki_search_v1", editable = true }
 gsm8k-v1 = { path = "environments/gsm8k_v1", editable = true }
 code-golf-v1 = { path = "environments/code_golf_v1", editable = true }
 reverse-text-v1 = { path = "environments/reverse_text_v1", editable = true }
-reverse-text = { path = "environments/reverse_text", editable = true }
 wordle-v1 = { path = "environments/wordle_v1", editable = true }
 color-codeword-v1 = { path = "environments/color_codeword_v1", editable = true }
 alphabet-sort-v1 = { path = "environments/alphabet_sort_v1", editable = true }
 scratchpad-v1 = { path = "environments/scratchpad_v1", editable = true }
-mmmu-v1 = { path = "environments/mmmu_v1", editable = true }
 
 [tool.uv.exclude-newer-package]
 # PrimeIntellect-published on PyPI (trusted publisher)

{verifiers-0.2.1.dev51 → verifiers-0.2.1.dev53}/tests/test_browser_env.py

@@ -452,98 +452,6 @@ class TestDOMModeLLMConfig:
         assert config is None
 
 
-# ============================================================================
-# Example Environment Tests
-# ============================================================================
-
-
-class TestExampleDatasets:
-    """Tests for example environment datasets."""
-
-    def test_dom_example_dataset_structure(self):
-        """Test DOM example dataset has correct structure."""
-        from environments.browser_dom_example.browser_dom_example import (
-            create_example_dataset,
-        )
-
-        dataset = create_example_dataset()
-
-        assert "question" in dataset.column_names
-        assert "answer" in dataset.column_names
-        assert "start_url" in dataset.column_names
-        assert "task_id" in dataset.column_names
-        assert len(dataset) >= 1
-
-    def test_cua_example_dataset_structure(self):
-        """Test CUA example dataset has correct structure."""
-        from environments.browser_cua_example.browser_cua_example import (
-            create_example_dataset,
-        )
-
-        dataset = create_example_dataset()
-
-        assert "question" in dataset.column_names
-        assert "answer" in dataset.column_names
-        assert "start_url" in dataset.column_names
-        assert "task_id" in dataset.column_names
-        assert len(dataset) >= 1
-
-
-class TestJudgeAnswer:
-    """Tests for judge_answer reward function."""
-
-    @pytest.mark.asyncio
-    async def test_judge_answer_returns_1_for_yes(self):
-        """Test that judge_answer returns 1.0 when judge says yes."""
-        from environments.browser_dom_example.browser_dom_example import judge_answer
-
-        async def mock_judge(prompt, completion, answer, state):
-            return "yes, the answer is correct"
-
-        result = await judge_answer(
-            judge=mock_judge,
-            prompt="What is 2+2?",
-            completion=[{"role": "assistant", "content": "The answer is 4"}],
-            answer="4",
-            state={},
-        )
-        assert result == 1.0
-
-    @pytest.mark.asyncio
-    async def test_judge_answer_returns_0_for_no(self):
-        """Test that judge_answer returns 0.0 when judge says no."""
-        from environments.browser_dom_example.browser_dom_example import judge_answer
-
-        async def mock_judge(prompt, completion, answer, state):
-            return "no, the answer is incorrect"
-
-        result = await judge_answer(
-            judge=mock_judge,
-            prompt="What is 2+2?",
-            completion=[{"role": "assistant", "content": "The answer is 5"}],
-            answer="4",
-            state={},
-        )
-        assert result == 0.0
-
-    @pytest.mark.asyncio
-    async def test_judge_answer_case_insensitive(self):
-        """Test that judge response check is case insensitive."""
-        from environments.browser_dom_example.browser_dom_example import judge_answer
-
-        async def mock_judge(prompt, completion, answer, state):
-            return "YES"
-
-        result = await judge_answer(
-            judge=mock_judge,
-            prompt="test",
-            completion="test",
-            answer="test",
-            state={},
-        )
-        assert result == 1.0
-
-
 # ============================================================================
 # Constants Tests
 # ============================================================================

{verifiers-0.2.1.dev51 → verifiers-0.2.1.dev53}/tests/test_install_utils.py

@@ -73,7 +73,7 @@ class TestIsHubEnv:
         assert is_hub_env("gsm8k") is False
 
     def test_local_env_relative_path(self):
-        assert is_hub_env("./environments/gsm8k") is False
+        assert is_hub_env("./environments/gsm8k_v1") is False
 
     def test_local_env_absolute_path(self):
         assert is_hub_env("/path/to/gsm8k") is False

{verifiers-0.2.1.dev51 → verifiers-0.2.1.dev53}/tests/v1/test_envs.py

@@ -1,10 +1,8 @@
-"""Smoke-eval every v1 example taskset in `environments/` through the `eval` CLI.
+"""Smoke-eval every V1 example taskset in `environments/` through the `eval` CLI.
 
-The v0 counterpart (`tests/test_envs.py`) covers v0 envs (`vf.load_environment` + `vf-eval`);
-here we run each `_v1` taskset with its required harness for one short, capped rollout and
-require it to succeed — so a broken example taskset fails CI. `compact` is excluded (it's a
-harness, not a taskset); SWE/container tasksets need a docker/prime runtime and are covered by
-the dedicated v1 e2e tests instead.
+Each `_v1` taskset runs with its required harness for one short, capped rollout, so a broken
+example taskset fails CI. `compact` is excluded (it's a harness, not a taskset);
+SWE/container tasksets need a docker/prime runtime and are covered by dedicated V1 e2e tests.
 """
 
 import os

{verifiers-0.2.1.dev51 → verifiers-0.2.1.dev53}/verifiers/envs/integrations/browser_env/README.md

@@ -139,16 +139,3 @@ CUA execution options:
 1. **Prebuilt image** (default): fastest startup
 2. **Binary upload** (`use_prebuilt_image=False`): custom server workflows
 3. **Manual local server** (`use_sandbox=False`): local development/debugging
-
-## Example Environments
-
-For complete reference implementations, see:
-
-- **DOM example:** `environments/browser_dom_example/`
-  - `environments/browser_dom_example/browser_dom_example.py`
-  - `environments/browser_dom_example/README.md`
-- **CUA example:** `environments/browser_cua_example/`
-  - `environments/browser_cua_example/browser_cua_example.py`
-  - `environments/browser_cua_example/README.md`
-
-These examples show end-to-end `load_environment()` setup, evaluation commands, and recommended runtime flags.

{verifiers-0.2.1.dev51 → verifiers-0.2.1.dev53}/verifiers/scripts/install.py

@@ -17,9 +17,9 @@ Usage:
 
 Examples:
     vf-install primeintellect/gsm8k        # Install from Hub
-    vf-install gsm8k                       # Install from ./environments/gsm8k
-    vf-install gsm8k -p /path/to/envs      # Install from custom local path
-    vf-install gsm8k -r                    # Install from GitHub repo
+    vf-install gsm8k-v1                    # Install from ./environments/gsm8k_v1
+    vf-install gsm8k-v1 -p /path/to/envs   # Install from custom local path
+    vf-install gsm8k-v1 -r                 # Install from GitHub repo
 """
 
 
@@ -30,9 +30,9 @@ def main():
         epilog="""
 Examples:
   vf-install primeintellect/gsm8k        Install from Environments Hub
-  vf-install gsm8k                       Install from ./environments/gsm8k
-  vf-install gsm8k -p /path/to/envs      Install from custom local path
-  vf-install gsm8k -r                    Install from GitHub repo
+  vf-install gsm8k-v1                    Install from ./environments/gsm8k_v1
+  vf-install gsm8k-v1 -p /path/to/envs   Install from custom local path
+  vf-install gsm8k-v1 -r                 Install from GitHub repo
         """,
     )
     parser.add_argument(

verifiers-0.2.1.dev51/tests/test_envs.py

@@ -1,195 +0,0 @@
-import os
-import subprocess
-from pathlib import Path
-
-import pytest
-import tomllib
-
-# Timeout in seconds for each subprocess step
-INSTALL_TIMEOUT = 600  # 10 minutes for venv creation + package install
-IMPORT_TIMEOUT = 120  # 2 minutes for importing a package
-LOAD_TIMEOUT = 300  # 5 minutes for loading an environment (may download datasets)
-EVAL_TIMEOUT = 600  # 10 minutes for running vf-eval with -n 1 -r 1
-
-SKIPPED_ENVS = [
-    # Requires fix for completion dataset setup
-    # uv run pytest tests/test_envs.py -vv -k continuation_quality
-    #
-    #     example_id = input_item["example_id"]
-    #                 ~~~~~~~~~~^^^^^^^^^^^^^^
-    # KeyError: 'example_id'
-    "continuation_quality",
-    # Different project structure (uses src/ layout, no pyproject.toml at root)
-    "mcp_env",
-    # Requires BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, MODEL_API_KEY
-    "browser_dom_example",
-    # Requires BROWSERBASE_API_KEY, BROWSERBASE_PROJECT_ID, and running CUA server
-    "browser_cua_example",
-    # Uses prime-tunnel which is still experimental and has low usage limits
-    "opencode_harbor",
-]
-
-SKIPPED_ENV_LOADING_ENVS = [
-    # OpenEnv datasets are built by resetting seeds in sandbox-backed env servers.
-    # Skip generic load checks here and cover via dedicated OpenEnv tests.
-    "openenv_echo",
-    "openenv_textarena",
-    # R2E-Gym pulls a full image-backed SWE taskset; cover it with dedicated v1 tests.
-    "rlm_swe_v1",
-]
-
-
-def get_environments() -> list[Path]:
-    """Get all subdirectories of `environments/`, or only changed environments if CHANGED_ENVS is set."""
-    all_envs = list(x for x in Path("environments").iterdir() if x.is_dir())
-
-    # Filter out skipped environments
-    all_envs = [env for env in all_envs if env.name not in SKIPPED_ENVS]
-
-    # These are v0 smoke tests (vf.load_environment + vf-eval). The v1 plugins (the `_v1`
-    # tasksets + the `compact` harness) are id-referenced and covered by tests/v1/test_envs.py.
-    all_envs = [
-        env
-        for env in all_envs
-        if not env.name.endswith("_v1") and env.name != "compact"
-    ]
-
-    # Filter environments if CHANGED_ENVS is set (for PRs)
-    changed_envs = os.getenv("CHANGED_ENVS")
-    if changed_envs == "none":
-        return []
-    if changed_envs:
-        changed_list = [e.strip() for e in changed_envs.split(",") if e.strip()]
-        if changed_list:
-            all_envs = [env for env in all_envs if env.name in changed_list]
-
-    return all_envs
-
-
-@pytest.mark.parametrize("env_dir", get_environments(), ids=lambda x: x.name)
-def test_pyproject_exists(env_dir: Path):
-    """Test that the pyproject.toml file exists for the given environment directory."""
-    assert (env_dir / "pyproject.toml").exists(), "pyproject.toml does not exist"
-
-
-@pytest.mark.parametrize("env_dir", get_environments(), ids=lambda x: x.name)
-def test_pyproject_has_metadata(env_dir: Path):
-    """Test that the pyproject.toml file has the required metadata."""
-    with open(env_dir / "pyproject.toml", "rb") as f:
-        pyproject = tomllib.load(f)
-    assert "name" in pyproject["project"], "pyproject.toml does not have a name"
-    assert "version" in pyproject["project"], "pyproject.toml does not have a version"
-    assert "description" in pyproject["project"], (
-        "pyproject.toml does not have a description"
-    )
-    assert pyproject["project"]["description"] != "Your environment description here", (
-        "Still uses placeholder description"
-    )
-    assert "tags" in pyproject["project"], "pyproject.toml does not have tags"
-    assert pyproject["project"]["tags"] != ["placeholder-tag", "train", "eval"], (
-        "Still uses placeholder tags"
-    )
-
-
-@pytest.mark.parametrize("env_dir", get_environments(), ids=lambda x: x.name)
-def test_readme_exists(env_dir: Path):
-    """Test that the README.md file exists for the given environment directory."""
-    assert (env_dir / "README.md").exists(), "README.md does not exist"
-
-
-@pytest.mark.slow
-@pytest.mark.parametrize("env_dir", get_environments(), ids=lambda x: x.name)
-def test_env(env_dir: Path, tmp_path_factory: pytest.TempPathFactory):
-    """Test environment in a fresh venv with local verifiers installed first."""
-    if env_dir.name in SKIPPED_ENVS:
-        pytest.skip(f"Skipping {env_dir.name}")
-    if env_dir.name in SKIPPED_ENV_LOADING_ENVS:
-        pytest.skip(f"Skipping dedicated-runtime smoke test for {env_dir.name}")
-    tmp_venv_dir = tmp_path_factory.mktemp(f"venv_{env_dir.name}")
-    repo_root = Path(__file__).parent.parent
-    cmd = (
-        f"cd {tmp_venv_dir} && uv venv --clear && source .venv/bin/activate && "
-        "uv pip install "
-        f"{repo_root.as_posix()} && "
-        "uv pip install "
-        f"{env_dir.absolute().as_posix()}"
-    )
-    try:
-        process = subprocess.run(
-            cmd,
-            shell=True,
-            executable="/bin/bash",
-            capture_output=True,
-            text=True,
-            timeout=INSTALL_TIMEOUT,
-        )
-    except subprocess.TimeoutExpired:
-        pytest.fail(f"Timed out after {INSTALL_TIMEOUT}s installing {env_dir.name}")
-    assert process.returncode == 0, (
-        f"Failed to create virtual environment: {process.stderr}"
-    )
-
-    help_test_can_import_env(tmp_venv_dir, env_dir)
-    help_test_can_load_env(tmp_venv_dir, env_dir)
-    help_test_can_eval_env(tmp_venv_dir, env_dir)
-
-
-def help_test_can_import_env(tmp_venv_dir: Path, env_dir: Path):
-    """Test that the environment can be imported as a package."""
-    import_cmd = f"cd {tmp_venv_dir} && source .venv/bin/activate && uv run python -c 'import {env_dir.name}'"
-    try:
-        process = subprocess.run(
-            import_cmd,
-            shell=True,
-            executable="/bin/bash",
-            capture_output=True,
-            text=True,
-            timeout=IMPORT_TIMEOUT,
-        )
-    except subprocess.TimeoutExpired:
-        pytest.fail(f"Timed out after {IMPORT_TIMEOUT}s importing {env_dir.name}")
-    assert process.returncode == 0, "Failed to import environment"
-
-
-def help_test_can_load_env(tmp_venv_dir: Path, env_dir: Path):
-    """Test that the environment can be loaded."""
-    load_cmd = f"""cd {tmp_venv_dir} && source .venv/bin/activate && uv run python -c 'import verifiers as vf; vf.load_environment("{env_dir.name}")'"""
-    try:
-        process = subprocess.run(
-            load_cmd,
-            shell=True,
-            executable="/bin/bash",
-            capture_output=True,
-            text=True,
-            timeout=LOAD_TIMEOUT,
-        )
-    except subprocess.TimeoutExpired:
-        pytest.fail(f"Timed out after {LOAD_TIMEOUT}s loading {env_dir.name}")
-    assert process.returncode == 0, "Failed to load environment"
-
-
-def help_test_can_eval_env(tmp_venv_dir: Path, env_dir: Path):
-    """Test that the environment can be run via vf-eval."""
-    if os.getenv("PRIME_API_KEY"):
-        model_flags = "-m openai/gpt-4.1-mini -b https://api.pinference.ai/api/v1 -k PRIME_API_KEY"
-    elif os.getenv("OPENAI_API_KEY"):
-        model_flags = "-m gpt-4.1-mini -b https://api.openai.com/v1 -k OPENAI_API_KEY"
-    else:
-        pytest.skip("Skipping vf-eval smoke test because no API key is configured")
-
-    eval_cmd = (
-        f"cd {tmp_venv_dir} && source .venv/bin/activate && "
-        f"uv run vf-eval {env_dir.name} {model_flags} -n 1 -r 1 -t 512"
-    )
-    try:
-        process = subprocess.run(
-            eval_cmd,
-            shell=True,
-            executable="/bin/bash",
-            capture_output=True,
-            text=True,
-            timeout=EVAL_TIMEOUT,
-        )
-    except subprocess.TimeoutExpired:
-        pytest.fail(f"Timed out after {EVAL_TIMEOUT}s evaluating {env_dir.name}")
-    assert process.returncode == 0, "Failed to evaluate environment"

verifiers-0.2.1.dev51/tests/test_wordle_env.py

@@ -1,22 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-
-def test_wordle_load_environment_reads_system_prompt_path(tmp_path, monkeypatch):
-    pytest.importorskip("nltk")
-    pytest.importorskip("textarena")
-    from environments.wordle import wordle
-
-    class CapturingTextArenaEnv:
-        def __init__(self, **kwargs):
-            self.kwargs = kwargs
-
-    prompt = "Optimized Wordle prompt.\n\nPreserve exact text.\n"
-    prompt_path = tmp_path / "system_prompt.txt"
-    prompt_path.write_text(prompt, encoding="utf-8")
-    monkeypatch.setattr(wordle, "TextArenaEnv", CapturingTextArenaEnv)
-
-    env = wordle.load_environment(path_to_system_prompt=Path(prompt_path))
-
-    assert env.kwargs["system_prompt"] == prompt

verifiers-0.2.1.dev51/verifiers/envs/experimental/composable/tasksets/search/README.md

@@ -1,42 +0,0 @@
-# Search Tasksets
-
-Composable search/research tasksets for agents that solve live information-seeking tasks in a sandbox.
-
-The search family is intentionally backend-oriented, mirroring the SWE taskset pattern while keeping the task contract research-centric: each task expects a single final answer rather than a code patch. Agents may use web/search tools, browser helpers, or other sandbox resources provided by the paired environment.
-
-## Backends
-
-| Backend | Source | Default dataset | Status |
-|---|---|---|---|
-| `openseeker` | [PolarSeeker/OpenSeeker](https://github.com/PolarSeeker/OpenSeeker) | [`PolarSeeker/OpenSeeker-v1-Data`](https://huggingface.co/datasets/PolarSeeker/OpenSeeker-v1-Data) | Binary semantic answer judge |
-| `quest` | [OSU-NLP-Group/QUEST](https://github.com/OSU-NLP-Group/QUEST) | [`osunlp/QUEST-RL-Data`](https://huggingface.co/datasets/osunlp/QUEST-RL-Data) | Objective tasks supported |
-| `redsearcher` | [RedSearchAgent/REDSearcher](https://github.com/RedSearchAgent/REDSearcher) | [`Zchu/REDSearcher_RL_1K`](https://huggingface.co/datasets/Zchu/REDSearcher_RL_1K) | Text RL query set supported |
-
-## Usage
-
-```python
-from verifiers.envs.experimental.composable.tasksets.search import make_search_taskset
-
-taskset = make_search_taskset(backend="openseeker")
-taskset = make_search_taskset(backend="quest", category="objective")
-redsearcher = make_search_taskset(
-    backend="redsearcher",
-    filter_fn="lambda x: x['info']['difficulty'] == 'easy'",
-)
-```
-
-`make_search_taskset()` dispatches by backend name. Unknown backends raise `ValueError` with the available backend list.
-
-## Output Contract
-
-Search tasksets should define their own output contract. The `quest`, `openseeker`, and `redsearcher` backends expect the agent to write one final researched response to `/task/answer.txt`, including supporting URLs/citations when available. Scratch reasoning, tool traces, and logs should not be written as the final answer.
-
-## Error Handling
-
-Search tasksets should use the framework error taxonomy for infrastructure failures:
-
-- `vf.SandboxError` for sandbox setup, command, or lifecycle failures.
-- `vf.ModelError` for judge/model provider failures.
-- `vf.InfraError` for dataset, evaluator, or external runtime failures.
-
-Incorrect answers should not set `state["error"]`; they should score normally, often as `0.0`.

verifiers-0.2.1.dev51/verifiers/envs/experimental/composable/tasksets/search/__init__.py

@@ -1,15 +0,0 @@
-"""Composable search/research tasksets."""
-
-from .search_tasksets import (
-    make_openseeker_taskset,
-    make_quest_taskset,
-    make_redsearcher_taskset,
-    make_search_taskset,
-)
-
-__all__ = [
-    "make_openseeker_taskset",
-    "make_quest_taskset",
-    "make_redsearcher_taskset",
-    "make_search_taskset",
-]

verifiers-0.2.1.dev51/verifiers/envs/experimental/composable/tasksets/search/openseeker/README.md

@@ -1,46 +0,0 @@
-# OpenSeeker Search Taskset
-
-Composable search taskset for [`PolarSeeker/OpenSeeker-v1-Data`](https://huggingface.co/datasets/PolarSeeker/OpenSeeker-v1-Data), the OpenSeeker v1 release associated with arXiv `2603.15594`.
-
-OpenSeeker v1 data contains synthesized deep-search QA pairs plus trajectories generated with `search` and `visit` tools. The public OpenSeeker evaluator scores only the final answer: it sends the question, gold answer, and model response to an LLM judge and expects `A` for correct or `B` for incorrect. This backend preserves that binary semantic answer-judge contract.
-
-By default, the taskset uses the full dataset. Use the shared `filter_fn`
-argument for row subsets such as source trajectory quality or tool-call count.
-The `trajectory_correctness` metadata describes the stored OpenSeeker source
-trajectory, not the validity of the question or gold answer.
-
-## Usage
-
-```python
-from verifiers.envs.experimental.composable.tasksets.search import make_search_taskset
-
-taskset = make_search_taskset(backend="openseeker")
-
-correct_source_trajectories = make_search_taskset(
-    backend="openseeker",
-    filter_fn="lambda x: x['info']['trajectory_correctness'] == 'Correct'",
-)
-
-shorter_source_trajectories = make_search_taskset(
-    backend="openseeker",
-    filter_fn="lambda x: (x['info']['number_of_tool_calls'] or 0) <= 20",
-)
-```
-
-## Arguments
-
-| Argument | Default | Description |
-|---|---:|---|
-| `dataset_name` | `PolarSeeker/OpenSeeker-v1-Data` | Hugging Face dataset name. |
-| `split` | `train` | Dataset split. |
-| `filter_fn` | `None` | Optional composable taskset filter over normalized rows. |
-| `include_trajectory` | `False` | Include the large source trajectory in task metadata. |
-| `answer_file` | `/task/answer.txt` | Final answer path in the sandbox. |
-| `judge_model` | `openai/gpt-5.4-mini` | OpenAI-compatible model used for binary answer judging. |
-| `judge_base_url` | `https://api.pinference.ai/api/v1` | Judge API base URL. |
-| `judge_api_key_var` | `PRIME_API_KEY` | Env var containing the judge API key. |
-| `judge_sampling_args` | `None` | Extra sampling args for judge calls. |
-
-## Output Contract
-
-Agents should write one final answer to `/task/answer.txt`. The answer should directly satisfy the question and may include supporting URLs/citations. The judge ignores citation verification and evaluates whether the final response semantically contains the gold answer without contradictions.

verifiers-0.2.1.dev51/verifiers/envs/experimental/composable/tasksets/search/openseeker/__init__.py

@@ -1,5 +0,0 @@
-"""OpenSeeker search taskset."""
-
-from .taskset import OpenSeekerRubric, OpenSeekerTaskSet
-
-__all__ = ["OpenSeekerRubric", "OpenSeekerTaskSet"]