verifiers 0.1.10.dev2__tar.gz → 0.1.10.dev4__tar.gz

{verifiers-0.1.10.dev2 → verifiers-0.1.10.dev4}/.gitignore

@@ -10,6 +10,7 @@ uv.lock
 .ropeproject/
 .scratch/
 .chroma_db/
+/.codex/environments/
 
 # artifacts
 core.* 

{verifiers-0.1.10.dev2 → verifiers-0.1.10.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.10.dev2
+Version: 0.1.10.dev4
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -32,8 +32,8 @@ Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: numpy
 Requires-Dist: openai-agents>=0.0.7
 Requires-Dist: openai>=1.108.1
-Requires-Dist: prime-sandboxes>=0.2.9
-Requires-Dist: prime-tunnel
+Requires-Dist: prime-sandboxes>=0.2.14
+Requires-Dist: prime-tunnel>=0.1.0
 Requires-Dist: pydantic>=2.11.9
 Requires-Dist: pyzmq>=27.1.0
 Requires-Dist: requests
@@ -47,19 +47,10 @@ Provides-Extra: browser
 Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
 Requires-Dist: stagehand>=3.0.0; extra == 'browser'
+Provides-Extra: openenv
+Requires-Dist: openenv-core[core]==0.2.1; extra == 'openenv'
 Provides-Extra: rg
 Requires-Dist: reasoning-gym; extra == 'rg'
-Provides-Extra: rl
-Requires-Dist: accelerate>=1.4.0; extra == 'rl'
-Requires-Dist: deepspeed>=0.17.6; extra == 'rl'
-Requires-Dist: flash-attn>=2.8.3; extra == 'rl'
-Requires-Dist: liger-kernel>=0.5.10; extra == 'rl'
-Requires-Dist: peft; extra == 'rl'
-Requires-Dist: requests; extra == 'rl'
-Requires-Dist: torch<2.9.0,>=2.8.0; extra == 'rl'
-Requires-Dist: transformers>=4.56.2; extra == 'rl'
-Requires-Dist: vllm<0.11.0,>=0.10.0; extra == 'rl'
-Requires-Dist: wandb; extra == 'rl'
 Provides-Extra: ta
 Requires-Dist: nltk; extra == 'ta'
 Requires-Dist: textarena; extra == 'ta'
@@ -140,8 +131,12 @@ prime lab setup
 This sets up a Python project if needed (with `uv init`), installs `verifiers` (with `uv add verifiers`), creates the recommended workspace structure, and downloads useful starter files:
 ```
 configs/
-├── endpoints.py        # OpenAI-compatible API endpoint configuration
-└── lab/                # Example configs for Hosted Training
+├── endpoints.toml      # OpenAI-compatible API endpoint configuration
+├── rl/                 # Example configs for Hosted Training
+├── eval/               # Example multi-environment eval configs
+└── gepa/               # Example configs for prompt optimization
+.prime/
+└── skills/             # Bundled workflow skills for create/browse/review/eval/GEPA/train/brainstorm
 environments/
 └── AGENTS.md           # Documentation for AI coding agents
 AGENTS.md               # Top-level documentation for AI coding agents
@@ -157,6 +152,14 @@ Environments built with Verifiers are self-contained Python modules. To initiali
 ```bash
 prime env init my-env # creates a new template in ./environments/my_env
 ```
+For OpenEnv integration, use:
+```bash
+prime env init my-openenv --openenv
+```
+Then copy your OpenEnv project into `environments/my_openenv/proj/` and build the image with:
+```bash
+uv run vf-build my-openenv
+```
 
 This will create a new module called `my_env` with a basic environment template.
 ```
@@ -195,7 +198,7 @@ To run a local evaluation with any OpenAI-compatible model, do:
 ```bash
 prime eval run my-env -m gpt-5-nano # run and save eval results locally
 ```
-Evaluations use [Prime Inference](https://docs.primeintellect.ai/inference/overview) by default; configure your own API endpoints in `./configs/endpoints.py`.
+Evaluations use [Prime Inference](https://docs.primeintellect.ai/inference/overview) by default; configure your own API endpoints in `./configs/endpoints.toml`.
 
 View local evaluation results in the terminal UI:
 ```bash

{verifiers-0.1.10.dev2 → verifiers-0.1.10.dev4}/README.md

@@ -73,8 +73,12 @@ prime lab setup
 This sets up a Python project if needed (with `uv init`), installs `verifiers` (with `uv add verifiers`), creates the recommended workspace structure, and downloads useful starter files:
 ```
 configs/
-├── endpoints.py        # OpenAI-compatible API endpoint configuration
-└── lab/                # Example configs for Hosted Training
+├── endpoints.toml      # OpenAI-compatible API endpoint configuration
+├── rl/                 # Example configs for Hosted Training
+├── eval/               # Example multi-environment eval configs
+└── gepa/               # Example configs for prompt optimization
+.prime/
+└── skills/             # Bundled workflow skills for create/browse/review/eval/GEPA/train/brainstorm
 environments/
 └── AGENTS.md           # Documentation for AI coding agents
 AGENTS.md               # Top-level documentation for AI coding agents
@@ -90,6 +94,14 @@ Environments built with Verifiers are self-contained Python modules. To initiali
 ```bash
 prime env init my-env # creates a new template in ./environments/my_env
 ```
+For OpenEnv integration, use:
+```bash
+prime env init my-openenv --openenv
+```
+Then copy your OpenEnv project into `environments/my_openenv/proj/` and build the image with:
+```bash
+uv run vf-build my-openenv
+```
 
 This will create a new module called `my_env` with a basic environment template.
 ```
@@ -128,7 +140,7 @@ To run a local evaluation with any OpenAI-compatible model, do:
 ```bash
 prime eval run my-env -m gpt-5-nano # run and save eval results locally
 ```
-Evaluations use [Prime Inference](https://docs.primeintellect.ai/inference/overview) by default; configure your own API endpoints in `./configs/endpoints.py`.
+Evaluations use [Prime Inference](https://docs.primeintellect.ai/inference/overview) by default; configure your own API endpoints in `./configs/endpoints.toml`.
 
 View local evaluation results in the terminal UI:
 ```bash

{verifiers-0.1.10.dev2 → verifiers-0.1.10.dev4}/pyproject.toml

@@ -36,8 +36,8 @@ dependencies = [
     "nest-asyncio>=1.6.0", # for jupyter notebooks
     "openai>=1.108.1",
     "openai-agents>=0.0.7",
-    "prime-tunnel",
-    "prime-sandboxes>=0.2.9",
+    "prime-tunnel>=0.1.0",
+    "prime-sandboxes>=0.2.14",
     "pydantic>=2.11.9",
     "requests",
     "rich",
@@ -64,6 +64,10 @@ dev = [
     "ipywidgets",
     "reasoning-gym",
     "textarena",
+    "openenv-core[core]==0.2.1",
+    "stagehand>=3.0.0",
+    "aiohttp>=3.9.0",
+    "python-dotenv>=1.0.0",
     "nltk",
 ]
 
@@ -75,40 +79,25 @@ ta = [
     "textarena",
     "nltk",
 ]
+openenv = [
+    "openenv-core[core]==0.2.1",
+]
 browser = [
     "stagehand>=3.0.0",
     "aiohttp>=3.9.0",
     "python-dotenv>=1.0.0",
 ]
-rl = [
-    "torch>=2.8.0,<2.9.0",
-    "transformers>=4.56.2",
-    "accelerate>=1.4.0",
-    "requests",
-    "peft",
-    "wandb",
-    "vllm>=0.10.0,<0.11.0",
-    "liger-kernel>=0.5.10",
-    "deepspeed>=0.17.6",
-    "flash-attn>=2.8.3",
-]
-
-[tool.uv.extra-build-dependencies]
-flash-attn = [{ requirement = "torch", match-runtime = true }]
-
-[tool.uv.extra-build-variables]
-flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE" }
-
 [project.scripts]
 vf-eval = "verifiers.scripts.eval:main"
 vf-gepa = "verifiers.scripts.gepa:main"
 vf-init = "verifiers.scripts.init:main"
 vf-install = "verifiers.scripts.install:main"
 vf-setup = "verifiers.scripts.setup:main"
+vf-build = "verifiers.scripts.build:main"
 vf-rl = "verifiers.scripts.rl:main"
 vf-train = "verifiers.scripts.train:main"
 vf-tui = "verifiers.scripts.tui:main"
-vf-vllm = "verifiers.rl.inference.server:main"
+vf-vllm = "verifiers.scripts.vllm:main"
 prime-rl = "verifiers.scripts.prime_rl:main"
 
 # hatchling configuration
@@ -171,9 +160,12 @@ filterwarnings = [
 asyncio_mode = "auto"
 norecursedirs = [".git", ".tox", "dist", "build", "*.egg", "__pycache__"]
 
+[tool.ty.environment]
+python-version = "3.13"
+
 [tool.ty.rules]
-unresolved-import = "warn"
 unknown-argument = "warn"
+redundant-cast = "ignore"
 
 [tool.ty.src]
 exclude = ["environments"]

verifiers-0.1.10.dev4/tests/test_client_config.py

@@ -0,0 +1,52 @@
+import pytest
+from pydantic import ValidationError
+
+from verifiers.types import ClientConfig, EndpointClientConfig
+
+
+def test_client_config_allows_leaf_endpoint_configs():
+    config = ClientConfig(
+        api_base_url="http://localhost:8000/v1",
+        endpoint_configs=[
+            EndpointClientConfig(api_base_url="http://localhost:8001/v1"),
+            {"api_base_url": "http://localhost:8002/v1"},
+        ],
+    )
+
+    assert len(config.endpoint_configs) == 2
+    assert config.endpoint_configs[0].api_base_url == "http://localhost:8001/v1"
+    assert config.endpoint_configs[1].api_base_url == "http://localhost:8002/v1"
+
+
+def test_client_config_rejects_recursive_endpoint_configs():
+    with pytest.raises(ValidationError, match="cannot include endpoint_configs"):
+        ClientConfig.model_validate(
+            {
+                "api_base_url": "http://localhost:8000/v1",
+                "endpoint_configs": [
+                    {
+                        "api_base_url": "http://localhost:8001/v1",
+                        "endpoint_configs": [
+                            {"api_base_url": "http://localhost:8002/v1"}
+                        ],
+                    }
+                ],
+            }
+        )
+
+
+def test_client_config_accepts_empty_nested_endpoint_configs_key():
+    config = ClientConfig.model_validate(
+        {
+            "api_base_url": "http://localhost:8000/v1",
+            "endpoint_configs": [
+                {
+                    "api_base_url": "http://localhost:8001/v1",
+                    "endpoint_configs": [],
+                }
+            ],
+        }
+    )
+
+    assert len(config.endpoint_configs) == 1
+    assert config.endpoint_configs[0].api_base_url == "http://localhost:8001/v1"

verifiers-0.1.10.dev4/tests/test_endpoint_registry.py

@@ -0,0 +1,177 @@
+from pathlib import Path
+
+from verifiers.utils.eval_utils import load_endpoints
+
+
+def test_load_endpoints_python_registry_normalizes_to_lists(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.py"
+    registry_path.write_text(
+        "ENDPOINTS = {\n"
+        '    "gpt-4.1-mini": {"model": "gpt-4.1-mini", "url": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"},\n'
+        "}\n",
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert set(endpoints.keys()) == {"gpt-4.1-mini"}
+    assert len(endpoints["gpt-4.1-mini"]) == 1
+    endpoint = endpoints["gpt-4.1-mini"][0]
+    assert endpoint["model"] == "gpt-4.1-mini"
+    assert endpoint["url"] == "https://api.openai.com/v1"
+    assert endpoint["key"] == "OPENAI_API_KEY"
+
+
+def test_load_endpoints_toml_groups_variants_by_endpoint_id(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "gpt-5-mini"\n'
+        'model = "openai/gpt-5-mini"\n'
+        'url = "https://api.pinference.ai/api/v1"\n'
+        'key = "PRIME_API_KEY"\n'
+        "\n"
+        "[[endpoint]]\n"
+        'endpoint_id = "gpt-5-mini"\n'
+        'model = "openai/gpt-5-mini"\n'
+        'url = "https://api.openai.com/v1"\n'
+        'key = "OPENAI_API_KEY"\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert set(endpoints.keys()) == {"gpt-5-mini"}
+    assert len(endpoints["gpt-5-mini"]) == 2
+    assert endpoints["gpt-5-mini"][0]["url"] == "https://api.pinference.ai/api/v1"
+    assert endpoints["gpt-5-mini"][1]["url"] == "https://api.openai.com/v1"
+
+
+def test_load_endpoints_toml_accepts_long_field_names(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "gpt-5-mini"\n'
+        'model = "openai/gpt-5-mini"\n'
+        'api_base_url = "https://api.pinference.ai/api/v1"\n'
+        'api_key_var = "PRIME_API_KEY"\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints["gpt-5-mini"][0]["url"] == "https://api.pinference.ai/api/v1"
+    assert endpoints["gpt-5-mini"][0]["key"] == "PRIME_API_KEY"
+
+
+def test_load_endpoints_toml_accepts_matching_short_and_long_fields(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "gpt-5-mini"\n'
+        'model = "openai/gpt-5-mini"\n'
+        'url = "https://api.pinference.ai/api/v1"\n'
+        'api_base_url = "https://api.pinference.ai/api/v1"\n'
+        'key = "PRIME_API_KEY"\n'
+        'api_key_var = "PRIME_API_KEY"\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints["gpt-5-mini"][0]["url"] == "https://api.pinference.ai/api/v1"
+    assert endpoints["gpt-5-mini"][0]["key"] == "PRIME_API_KEY"
+
+
+def test_load_endpoints_toml_rejects_conflicting_url_fields(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "gpt-5-mini"\n'
+        'model = "openai/gpt-5-mini"\n'
+        'url = "https://a.example/v1"\n'
+        'api_base_url = "https://b.example/v1"\n'
+        'key = "PRIME_API_KEY"\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints == {}
+
+
+def test_load_endpoints_toml_rejects_conflicting_key_fields(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.toml"
+    registry_path.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "gpt-5-mini"\n'
+        'model = "openai/gpt-5-mini"\n'
+        'url = "https://a.example/v1"\n'
+        'key = "A_KEY"\n'
+        'api_key_var = "B_KEY"\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert endpoints == {}
+
+
+def test_load_endpoints_python_registry_supports_list_variants(tmp_path: Path):
+    registry_path = tmp_path / "endpoints.py"
+    registry_path.write_text(
+        "ENDPOINTS = {\n"
+        '    "gpt-5-mini": [\n'
+        '        {"model": "gpt-5-mini", "url": "https://a.example/v1", "key": "A_KEY"},\n'
+        '        {"model": "gpt-5-mini", "url": "https://b.example/v1", "key": "A_KEY"},\n'
+        "    ]\n"
+        "}\n",
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(registry_path))
+
+    assert set(endpoints.keys()) == {"gpt-5-mini"}
+    assert len(endpoints["gpt-5-mini"]) == 2
+    assert endpoints["gpt-5-mini"][0]["url"] == "https://a.example/v1"
+    assert endpoints["gpt-5-mini"][1]["url"] == "https://b.example/v1"
+
+
+def test_load_endpoints_directory_prefers_toml_then_python(tmp_path: Path):
+    python_registry = tmp_path / "endpoints.py"
+    toml_registry = tmp_path / "endpoints.toml"
+
+    python_registry.write_text(
+        "ENDPOINTS = {\n"
+        '    "from-py": {"model": "m", "url": "https://py.example/v1", "key": "PY_KEY"},\n'
+        "}\n",
+        encoding="utf-8",
+    )
+    toml_registry.write_text(
+        "[[endpoint]]\n"
+        'endpoint_id = "from-toml"\n'
+        'model = "m"\n'
+        'url = "https://toml.example/v1"\n'
+        'key = "TOML_KEY"\n',
+        encoding="utf-8",
+    )
+
+    endpoints = load_endpoints(str(tmp_path))
+    assert set(endpoints.keys()) == {"from-toml"}
+
+    toml_registry.unlink()
+    endpoints = load_endpoints(str(tmp_path))
+    assert set(endpoints.keys()) == {"from-py"}
+
+
+def test_qwen3_vl_endpoint_ids_map_to_vl_models():
+    endpoints = load_endpoints("./configs/endpoints.toml")
+
+    assert endpoints["qwen3-vl-30b-i"][0]["model"] == "qwen/qwen3-vl-30b-a3b-instruct"
+    assert endpoints["qwen3-vl-30b-t"][0]["model"] == "qwen/qwen3-vl-30b-a3b-thinking"
+    assert (
+        endpoints["qwen3-vl-235b-i"][0]["model"] == "qwen/qwen3-vl-235b-a22b-instruct"
+    )
+    assert (
+        endpoints["qwen3-vl-235b-t"][0]["model"] == "qwen/qwen3-vl-235b-a22b-thinking"
+    )