verifiers 0.1.15.dev10__tar.gz → 0.1.15.dev12__tar.gz

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/.gitignore

@@ -43,8 +43,6 @@ logs/
 .chroma_db/
 
 # development
-packages/harnesses
-packages/tasksets
 scratch/
 .idea/
 .vscode/

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev10
+Version: 0.1.15.dev12
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -53,10 +53,20 @@ Provides-Extra: browser
 Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
 Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
 Requires-Dist: stagehand>=3.0.0; extra == 'browser'
+Provides-Extra: harnesses
+Requires-Dist: harnesses>=0.1.1; extra == 'harnesses'
+Provides-Extra: nemogym
+Requires-Dist: harnesses[nemogym]>=0.1.1; extra == 'nemogym'
+Requires-Dist: tasksets[nemogym]>=0.1.1; extra == 'nemogym'
 Provides-Extra: openenv
-Requires-Dist: openenv-core>=0.3.0; extra == 'openenv'
+Requires-Dist: tasksets[openenv]>=0.1.1; extra == 'openenv'
+Provides-Extra: openreward
+Requires-Dist: tasksets[openreward]>=0.1.1; extra == 'openreward'
+Provides-Extra: packages
+Requires-Dist: harnesses>=0.1.1; extra == 'packages'
+Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.1; extra == 'packages'
 Provides-Extra: renderers
-Requires-Dist: renderers>=0.1.8.dev4; extra == 'renderers'
+Requires-Dist: renderers>=0.1.8.dev28; extra == 'renderers'
 Provides-Extra: rg
 Requires-Dist: reasoning-gym; extra == 'rg'
 Provides-Extra: rl
@@ -73,6 +83,8 @@ Requires-Dist: wandb; extra == 'rl'
 Provides-Extra: ta
 Requires-Dist: nltk; extra == 'ta'
 Requires-Dist: textarena; extra == 'ta'
+Provides-Extra: tasksets
+Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.1; extra == 'tasksets'
 Description-Content-Type: text/markdown
 
 <p align="center">
@@ -220,19 +232,13 @@ custom harnesses, use the v1 Taskset/Harness path:
 # my_env.py
 import verifiers as vf
 
-@vf.reward(weight=1.0)
-async def contains_answer(task, state) -> float:
-    return float(task["answer"] in str(state.get("completion") or ""))
 
 class MyTasksetConfig(vf.TasksetConfig):
     split: str = "train"
 
 
-class MyTaskset(vf.Taskset):
-    config: MyTasksetConfig
-    _default_rewards = (contains_answer,)
-
-    def rows(self) -> list[dict[str, object]]:
+class MyTaskset(vf.Taskset[MyTasksetConfig]):
+    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
         rows = [
             {
                 "prompt": [{"role": "user", "content": "Reverse abc."}],
@@ -243,28 +249,32 @@ class MyTaskset(vf.Taskset):
         ]
         return [row for row in rows if row["split"] == self.config.split]
 
+    @vf.reward(weight=1.0)
+    async def contains_answer(self, task, state) -> float:
+        return float(task["answer"] in str(state.get("completion") or ""))
+
 
 def load_taskset(config: MyTasksetConfig) -> MyTaskset:
-    assert isinstance(config, MyTasksetConfig)
     return MyTaskset(config=config)
 
 
 def load_environment(config: vf.EnvConfig) -> vf.Env:
-    taskset_config = config.taskset
-    assert isinstance(taskset_config, MyTasksetConfig)
-    return vf.Env(taskset=load_taskset(taskset_config))
+    return vf.Env(taskset=vf.load_taskset(config=config.taskset))
 ```
 If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
-Reusable taskset and harness packages live under `verifiers.v1.packages` while
-the v1 API stabilizes, and are re-exported from `verifiers.v1` for normal use.
-For example, Harbor task directories can run through the bundled OpenCode CLI
-harness with:
+Reusable taskset and harness packages live in `tasksets` and `harnesses`.
+Install them with `uv add "verifiers[packages]"`, or with the narrower
+`verifiers[tasksets]` and `verifiers[harnesses]` extras. For example, Harbor
+task directories can run through the bundled OpenCode CLI harness with:
 
 ```python
+from harnesses import OpenCode, OpenCodeConfig
+from tasksets import HarborTaskset, HarborTasksetConfig
+
 env = vf.Env(
-    taskset=vf.HarborTaskset(config=vf.HarborTasksetConfig()),
-    harness=vf.OpenCode(config=vf.OpenCodeConfig()),
+    taskset=HarborTaskset(config=HarborTasksetConfig(bundle_package=__name__)),
+    harness=OpenCode(config=OpenCodeConfig()),
 )
 ```
 

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/README.md

@@ -143,19 +143,13 @@ custom harnesses, use the v1 Taskset/Harness path:
 # my_env.py
 import verifiers as vf
 
-@vf.reward(weight=1.0)
-async def contains_answer(task, state) -> float:
-    return float(task["answer"] in str(state.get("completion") or ""))
 
 class MyTasksetConfig(vf.TasksetConfig):
     split: str = "train"
 
 
-class MyTaskset(vf.Taskset):
-    config: MyTasksetConfig
-    _default_rewards = (contains_answer,)
-
-    def rows(self) -> list[dict[str, object]]:
+class MyTaskset(vf.Taskset[MyTasksetConfig]):
+    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
         rows = [
             {
                 "prompt": [{"role": "user", "content": "Reverse abc."}],
@@ -166,28 +160,32 @@ class MyTaskset(vf.Taskset):
         ]
         return [row for row in rows if row["split"] == self.config.split]
 
+    @vf.reward(weight=1.0)
+    async def contains_answer(self, task, state) -> float:
+        return float(task["answer"] in str(state.get("completion") or ""))
+
 
 def load_taskset(config: MyTasksetConfig) -> MyTaskset:
-    assert isinstance(config, MyTasksetConfig)
     return MyTaskset(config=config)
 
 
 def load_environment(config: vf.EnvConfig) -> vf.Env:
-    taskset_config = config.taskset
-    assert isinstance(taskset_config, MyTasksetConfig)
-    return vf.Env(taskset=load_taskset(taskset_config))
+    return vf.Env(taskset=vf.load_taskset(config=config.taskset))
 ```
 If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
-Reusable taskset and harness packages live under `verifiers.v1.packages` while
-the v1 API stabilizes, and are re-exported from `verifiers.v1` for normal use.
-For example, Harbor task directories can run through the bundled OpenCode CLI
-harness with:
+Reusable taskset and harness packages live in `tasksets` and `harnesses`.
+Install them with `uv add "verifiers[packages]"`, or with the narrower
+`verifiers[tasksets]` and `verifiers[harnesses]` extras. For example, Harbor
+task directories can run through the bundled OpenCode CLI harness with:
 
 ```python
+from harnesses import OpenCode, OpenCodeConfig
+from tasksets import HarborTaskset, HarborTasksetConfig
+
 env = vf.Env(
-    taskset=vf.HarborTaskset(config=vf.HarborTasksetConfig()),
-    harness=vf.OpenCode(config=vf.OpenCodeConfig()),
+    taskset=HarborTaskset(config=HarborTasksetConfig(bundle_package=__name__)),
+    harness=OpenCode(config=OpenCodeConfig()),
 )
 ```
 

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/pyproject.toml

@@ -69,12 +69,12 @@ dev = [
     "ipykernel",
     "ipywidgets",
     "reasoning-gym",
-    "textarena",
     "stagehand>=3.0.0",
     "aiohttp>=3.9.0",
     "python-dotenv>=1.0.0",
-    "nltk",
-    "renderers>=0.1.8.dev4",
+    "harnesses",
+    "renderers>=0.1.8.dev28",
+    "tasksets[openenv,openreward,ta]",
 ]
 policy = [
     "semgrep>=1.150.0",
@@ -85,8 +85,18 @@ rg = [
     "reasoning-gym",
 ]
 ta = [
-    "textarena",
     "nltk",
+    "textarena",
+]
+tasksets = [
+    "tasksets[openenv,openreward,ta]>=0.1.1",
+]
+harnesses = [
+    "harnesses>=0.1.1",
+]
+packages = [
+    "harnesses>=0.1.1",
+    "tasksets[openenv,openreward,ta]>=0.1.1",
 ]
 browser = [
     "stagehand>=3.0.0",
@@ -94,10 +104,17 @@ browser = [
     "python-dotenv>=1.0.0",
 ]
 openenv = [
-    "openenv-core>=0.3.0",
+    "tasksets[openenv]>=0.1.1",
+]
+openreward = [
+    "tasksets[openreward]>=0.1.1",
+]
+nemogym = [
+    "harnesses[nemogym]>=0.1.1",
+    "tasksets[nemogym]>=0.1.1",
 ]
 renderers = [
-    "renderers>=0.1.8.dev4",
+    "renderers>=0.1.8.dev28",
 ]
 rl = [
     "torch>=2.8.0,<2.9.0",
@@ -121,12 +138,53 @@ conflicts = [
         { extra = "openenv" },
         { group = "policy" },
     ],
+    [
+        { extra = "nemogym" },
+        { extra = "openenv" },
+    ],
+    [
+        { extra = "nemogym" },
+        { group = "dev" },
+    ],
+    [
+        { extra = "nemogym" },
+        { extra = "tasksets" },
+    ],
+    [
+        { extra = "nemogym" },
+        { extra = "packages" },
+    ],
+    [
+        { extra = "ta" },
+        { group = "policy" },
+    ],
+    [
+        { extra = "tasksets" },
+        { group = "policy" },
+    ],
+    [
+        { extra = "packages" },
+        { group = "policy" },
+    ],
+    [
+        { extra = "openreward" },
+        { group = "policy" },
+    ],
+    [
+        { group = "dev" },
+        { group = "policy" },
+    ],
 ]
+
 [[tool.uv.index]]
 name = "pypi"
 url = "https://pypi.org/simple"
 default = true
 
+[tool.uv.sources]
+harnesses = { path = "packages/harnesses", editable = true }
+tasksets = { path = "packages/tasksets", editable = true }
+
 [tool.uv.exclude-newer-package]
 # PrimeIntellect-published on PyPI (trusted publisher)
 prime-tunnel = false
@@ -134,6 +192,8 @@ prime-sandboxes = false
 prime-pydantic-config = false
 renderers = false
 openenv-core = false
+harnesses = false
+tasksets = false
 
 [tool.uv.extra-build-dependencies]
 flash-attn = [{ requirement = "torch", match-runtime = true }]

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/tests/test_endpoint_registry.py

@@ -58,8 +58,8 @@ def test_load_endpoints_toml_groups_variants_by_endpoint_id(tmp_path: Path):
 
     assert set(endpoints.keys()) == {"gpt-5-mini"}
     assert len(endpoints["gpt-5-mini"]) == 2
-    assert endpoints["gpt-5-mini"][0]["url"] == "https://api.pinference.ai/api/v1"
-    assert endpoints["gpt-5-mini"][1]["url"] == "https://api.openai.com/v1"
+    assert endpoints["gpt-5-mini"][0].base_url == "https://api.pinference.ai/api/v1"
+    assert endpoints["gpt-5-mini"][1].base_url == "https://api.openai.com/v1"
 
 
 def test_load_endpoints_toml_accepts_long_field_names(tmp_path: Path):
@@ -75,8 +75,8 @@ def test_load_endpoints_toml_accepts_long_field_names(tmp_path: Path):
 
     endpoints = load_endpoints(str(registry_path))
 
-    assert endpoints["gpt-5-mini"][0]["url"] == "https://api.pinference.ai/api/v1"
-    assert endpoints["gpt-5-mini"][0]["key"] == "PRIME_API_KEY"
+    assert endpoints["gpt-5-mini"][0].base_url == "https://api.pinference.ai/api/v1"
+    assert endpoints["gpt-5-mini"][0].api_key_var == "PRIME_API_KEY"
 
 
 def test_load_endpoints_toml_accepts_matching_short_and_long_fields(tmp_path: Path):
@@ -94,8 +94,8 @@ def test_load_endpoints_toml_accepts_matching_short_and_long_fields(tmp_path: Pa
 
     endpoints = load_endpoints(str(registry_path))
 
-    assert endpoints["gpt-5-mini"][0]["url"] == "https://api.pinference.ai/api/v1"
-    assert endpoints["gpt-5-mini"][0]["key"] == "PRIME_API_KEY"
+    assert endpoints["gpt-5-mini"][0].base_url == "https://api.pinference.ai/api/v1"
+    assert endpoints["gpt-5-mini"][0].api_key_var == "PRIME_API_KEY"
 
 
 def test_load_endpoints_toml_rejects_conflicting_url_fields(tmp_path: Path):
@@ -165,14 +165,10 @@ def test_load_endpoints_directory_uses_toml_and_warns_on_ignored_python(
 def test_qwen3_vl_endpoint_ids_map_to_vl_models():
     endpoints = load_endpoints("./configs/endpoints.toml")
 
-    assert endpoints["qwen3-vl-30b-i"][0]["model"] == "qwen/qwen3-vl-30b-a3b-instruct"
-    assert endpoints["qwen3-vl-30b-t"][0]["model"] == "qwen/qwen3-vl-30b-a3b-thinking"
-    assert (
-        endpoints["qwen3-vl-235b-i"][0]["model"] == "qwen/qwen3-vl-235b-a22b-instruct"
-    )
-    assert (
-        endpoints["qwen3-vl-235b-t"][0]["model"] == "qwen/qwen3-vl-235b-a22b-thinking"
-    )
+    assert endpoints["qwen3-vl-30b-i"][0].model == "qwen/qwen3-vl-30b-a3b-instruct"
+    assert endpoints["qwen3-vl-30b-t"][0].model == "qwen/qwen3-vl-30b-a3b-thinking"
+    assert endpoints["qwen3-vl-235b-i"][0].model == "qwen/qwen3-vl-235b-a22b-instruct"
+    assert endpoints["qwen3-vl-235b-t"][0].model == "qwen/qwen3-vl-235b-a22b-thinking"
 
 
 def test_load_endpoints_toml_accepts_type_shorthand(tmp_path: Path):
@@ -189,7 +185,7 @@ def test_load_endpoints_toml_accepts_type_shorthand(tmp_path: Path):
 
     endpoints = load_endpoints(str(registry_path))
 
-    assert endpoints["haiku"][0]["api_client_type"] == "anthropic_messages"
+    assert endpoints["haiku"][0].api_client_type == "anthropic_messages"
 
 
 def test_load_endpoints_toml_accepts_openai_responses_type(tmp_path: Path):
@@ -206,7 +202,7 @@ def test_load_endpoints_toml_accepts_openai_responses_type(tmp_path: Path):
 
     endpoints = load_endpoints(str(registry_path))
 
-    assert endpoints["gpt-responses"][0]["api_client_type"] == "openai_responses"
+    assert endpoints["gpt-responses"][0].api_client_type == "openai_responses"
 
 
 def test_load_endpoints_toml_accepts_headers_table(tmp_path: Path):
@@ -223,7 +219,7 @@ def test_load_endpoints_toml_accepts_headers_table(tmp_path: Path):
 
     endpoints = load_endpoints(str(registry_path))
 
-    assert endpoints["proxy"][0]["extra_headers"] == {"X-Custom": "v1"}
+    assert endpoints["proxy"][0].extra_headers == {"X-Custom": "v1"}
 
 
 def test_load_endpoints_toml_accepts_extra_headers_alias(tmp_path: Path):
@@ -240,7 +236,7 @@ def test_load_endpoints_toml_accepts_extra_headers_alias(tmp_path: Path):
 
     endpoints = load_endpoints(str(registry_path))
 
-    assert endpoints["proxy"][0]["extra_headers"] == {"X-A": "a"}
+    assert endpoints["proxy"][0].extra_headers == {"X-A": "a"}
 
 
 def test_load_endpoints_toml_rejects_headers_and_extra_headers_together(

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/tests/test_envs.py

@@ -158,6 +158,10 @@ def test_env(env_dir: Path, tmp_path_factory: pytest.TempPathFactory):
         f"{repo_root.as_posix()} && "
         "uv pip install "
         "--exclude-newer-package prime-pydantic-config=2026-05-20T00:00:00Z "
+        f"{(repo_root / 'packages' / 'tasksets').as_posix()} "
+        f"{(repo_root / 'packages' / 'harnesses').as_posix()} && "
+        "uv pip install "
+        "--exclude-newer-package prime-pydantic-config=2026-05-20T00:00:00Z "
         f"{env_dir.absolute().as_posix()}"
     )
     try:

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/tests/test_eval_cli.py

@@ -11,7 +11,7 @@ import pytest
 
 import verifiers.scripts.eval as vf_eval
 import verifiers.utils.eval_utils
-from verifiers.types import GenerateOutputs
+from verifiers.types import EndpointConfig, GenerateOutputs
 from verifiers.utils.eval_utils import load_toml_config
 from verifiers.utils.path_utils import get_eval_results_path
 from verifiers.utils.save_utils import states_to_outputs
@@ -21,6 +21,10 @@ def fail_load_endpoints(*_: object) -> dict:
     raise AssertionError("load_endpoints should not be called")
 
 
+def endpoint(**values: object) -> EndpointConfig:
+    return EndpointConfig.model_validate(values)
+
+
 @pytest.fixture
 def run_cli(make_metadata, make_state, make_input):
     def _run_cli(
@@ -322,12 +326,12 @@ def test_cli_registry_headers_merged_with_eval_toml(tmp_path, monkeypatch, run_c
         {"env_id_or_config": str(cfg)},
         endpoints={
             "gpt-5-mini": [
-                {
-                    "model": "gpt-5-mini",
-                    "url": "https://a.example/v1",
-                    "key": "OPENAI_API_KEY",
-                    "extra_headers": {"X-Reg": "r"},
-                }
+                endpoint(
+                    model="gpt-5-mini",
+                    base_url="https://a.example/v1",
+                    api_key_var="OPENAI_API_KEY",
+                    extra_headers={"X-Reg": "r"},
+                )
             ]
         },
     )
@@ -350,18 +354,18 @@ def test_cli_multi_variant_preserves_per_row_registry_headers(monkeypatch, run_c
         },
         endpoints={
             "gpt-5-mini": [
-                {
-                    "model": "gpt-5-mini",
-                    "url": "https://a.example/v1",
-                    "key": "OPENAI_API_KEY",
-                    "extra_headers": {"X-Row": "a"},
-                },
-                {
-                    "model": "gpt-5-mini",
-                    "url": "https://b.example/v1",
-                    "key": "OPENAI_API_KEY",
-                    "extra_headers": {"X-Row": "b"},
-                },
+                endpoint(
+                    model="gpt-5-mini",
+                    base_url="https://a.example/v1",
+                    api_key_var="OPENAI_API_KEY",
+                    extra_headers={"X-Row": "a"},
+                ),
+                endpoint(
+                    model="gpt-5-mini",
+                    base_url="https://b.example/v1",
+                    api_key_var="OPENAI_API_KEY",
+                    extra_headers={"X-Row": "b"},
+                ),
             ]
         },
     )
@@ -381,16 +385,16 @@ def test_cli_endpoint_alias_multi_variant_sets_multi_base_urls(monkeypatch, run_
         },
         endpoints={
             "gpt-5-mini": [
-                {
-                    "model": "gpt-5-mini",
-                    "url": "https://a.example/v1",
-                    "key": "OPENAI_API_KEY",
-                },
-                {
-                    "model": "gpt-5-mini",
-                    "url": "https://b.example/v1",
-                    "key": "OPENAI_API_KEY",
-                },
+                endpoint(
+                    model="gpt-5-mini",
+                    base_url="https://a.example/v1",
+                    api_key_var="OPENAI_API_KEY",
+                ),
+                endpoint(
+                    model="gpt-5-mini",
+                    base_url="https://b.example/v1",
+                    api_key_var="OPENAI_API_KEY",
+                ),
             ]
         },
     )
@@ -417,11 +421,11 @@ def test_cli_model_flag_resolves_endpoint_alias_when_registry_present(
         },
         endpoints={
             "gpt-4.1-mini": [
-                {
-                    "model": "openai/gpt-4.1-mini",
-                    "url": "https://alias.example/v1",
-                    "key": "ALIAS_API_KEY",
-                }
+                endpoint(
+                    model="openai/gpt-4.1-mini",
+                    base_url="https://alias.example/v1",
+                    api_key_var="ALIAS_API_KEY",
+                )
             ]
         },
     )
@@ -443,12 +447,12 @@ def test_cli_model_flag_uses_endpoint_client_type_when_provided(monkeypatch, run
         },
         endpoints={
             "haiku": [
-                {
-                    "model": "claude-haiku-4-5",
-                    "url": "https://api.anthropic.com",
-                    "key": "ANTHROPIC_API_KEY",
-                    "api_client_type": "anthropic_messages",
-                }
+                endpoint(
+                    model="claude-haiku-4-5",
+                    base_url="https://api.anthropic.com",
+                    api_key_var="ANTHROPIC_API_KEY",
+                    api_client_type="anthropic_messages",
+                )
             ]
         },
     )
@@ -518,16 +522,16 @@ def test_cli_endpoint_alias_multi_variant_supports_mixed_keys(monkeypatch, run_c
         },
         endpoints={
             "gpt-5-mini": [
-                {
-                    "model": "gpt-5-mini",
-                    "url": "https://a.example/v1",
-                    "key": "PRIME_API_KEY",
-                },
-                {
-                    "model": "gpt-5-mini",
-                    "url": "https://b.example/v1",
-                    "key": "OPENAI_API_KEY",
-                },
+                endpoint(
+                    model="gpt-5-mini",
+                    base_url="https://a.example/v1",
+                    api_key_var="PRIME_API_KEY",
+                ),
+                endpoint(
+                    model="gpt-5-mini",
+                    base_url="https://b.example/v1",
+                    api_key_var="OPENAI_API_KEY",
+                ),
             ]
         },
     )
@@ -551,16 +555,16 @@ def test_cli_endpoint_id_resolves_registry_alias(monkeypatch, run_cli):
             },
             endpoints={
                 "gpt-5-mini": [
-                    {
-                        "model": "gpt-5-mini",
-                        "url": "https://a.example/v1",
-                        "key": "OPENAI_API_KEY",
-                    },
-                    {
-                        "model": "gpt-5-mini",
-                        "url": "https://b.example/v1",
-                        "key": "OPENAI_API_KEY",
-                    },
+                    endpoint(
+                        model="gpt-5-mini",
+                        base_url="https://a.example/v1",
+                        api_key_var="OPENAI_API_KEY",
+                    ),
+                    endpoint(
+                        model="gpt-5-mini",
+                        base_url="https://b.example/v1",
+                        api_key_var="OPENAI_API_KEY",
+                    ),
                 ]
             },
         )
@@ -602,11 +606,11 @@ def test_cli_endpoint_id_accepts_directory_endpoints_path(monkeypatch, run_cli):
                 },
                 endpoints={
                     "gpt-5-mini": [
-                        {
-                            "model": "gpt-5-mini",
-                            "url": "https://a.example/v1",
-                            "key": "OPENAI_API_KEY",
-                        }
+                        endpoint(
+                            model="gpt-5-mini",
+                            base_url="https://a.example/v1",
+                            api_key_var="OPENAI_API_KEY",
+                        )
                     ]
                 },
             )
@@ -647,11 +651,11 @@ def test_cli_endpoint_id_requires_toml_endpoints_path(monkeypatch, run_cli):
                 },
                 endpoints={
                     "gpt-5-mini": [
-                        {
-                            "model": "gpt-5-mini",
-                            "url": "https://a.example/v1",
-                            "key": "OPENAI_API_KEY",
-                        }
+                        endpoint(
+                            model="gpt-5-mini",
+                            base_url="https://a.example/v1",
+                            api_key_var="OPENAI_API_KEY",
+                        )
                     ]
                 },
             )

{verifiers-0.1.15.dev10 → verifiers-0.1.15.dev12}/tests/test_gepa_cli.py

@@ -11,14 +11,25 @@ from verifiers.scripts.gepa import (
     load_gepa_toml_config,
     resolve_gepa_config_args,
 )
+from verifiers.types import EndpointConfig
 
 
 def test_gepa_extra_headers_from_group_requires_consistent_variants():
     with pytest.raises(ValueError, match="different headers"):
         _gepa_extra_headers_from_group(
             [
-                {"extra_headers": {"X-A": "1"}},
-                {"extra_headers": {"X-A": "2"}},
+                EndpointConfig(
+                    api_key_var="K",
+                    base_url="https://a.example/v1",
+                    model="m",
+                    extra_headers={"X-A": "1"},
+                ),
+                EndpointConfig(
+                    api_key_var="K",
+                    base_url="https://a.example/v1",
+                    model="m",
+                    extra_headers={"X-A": "2"},
+                ),
             ],
             "my-alias",
         )
@@ -27,8 +38,18 @@ def test_gepa_extra_headers_from_group_requires_consistent_variants():
 def test_gepa_extra_headers_from_group_returns_first_row_dict():
     h = _gepa_extra_headers_from_group(
         [
-            {"extra_headers": {"X-A": "x"}},
-            {"extra_headers": {"X-A": "x"}},
+            EndpointConfig(
+                api_key_var="K",
+                base_url="https://a.example/v1",
+                model="m",
+                extra_headers={"X-A": "x"},
+            ),
+            EndpointConfig(
+                api_key_var="K",
+                base_url="https://a.example/v1",
+                model="m",
+                extra_headers={"X-A": "x"},
+            ),
         ],
         "my-alias",
     )