verifiers 0.1.15.dev5__tar.gz → 0.1.15.dev7__tar.gz

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev5
+Version: 0.1.15.dev7
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -174,6 +174,10 @@ Environments built with Verifiers are self-contained Python modules. To initiali
 ```bash
 prime env init my-env # creates a new template in ./environments/my_env
 ```
+Add an explicit harness loader when the environment owns harness behavior:
+```bash
+prime env init my-env --with-harness
+```
 For OpenEnv integration, use:
 ```bash
 prime env init my-openenv --openenv
@@ -191,7 +195,9 @@ environments/my_env/
 └── README.md           # Documentation
 ```
 
-Environment modules should expose a `load_environment` function which returns an instance of the Environment object, and which can accept custom arguments. For example: 
+Environment modules should expose a `load_environment` function which returns an
+environment object. For simple legacy environments, this can still be a direct
+constructor:
 ```python
 # my_env.py
 import verifiers as vf
@@ -223,7 +229,7 @@ def source():
 async def contains_answer(task, state) -> float:
     return float(task["answer"] in str(state.get("completion") or ""))
 
-def load_taskset(config: vf.TasksetConfig | None = None):
+def load_taskset(config: vf.TasksetConfig):
     return vf.Taskset(source=source, rewards=[contains_answer], config=config)
 
 def load_environment(config: vf.EnvConfig) -> vf.Env:
@@ -244,8 +250,8 @@ env = vf.Env(
 ```
 
 The same environment package is the unit used by evals and `prime-rl`. The
-trainer owns model, endpoint, sampling, and rollout count; v1-specific taskset
-and harness options stay under `env.taskset` and `env.harness`:
+trainer owns model, endpoint, sampling, and rollout count; v1-specific options
+stay on the taskset or harness config that owns them:
 
 ```toml
 # configs/rl/my-v1-env.toml
@@ -260,12 +266,12 @@ max_tokens = 4096
 [[env]]
 id = "my-env"
 
-[env.args]
-arg1 = "non-th-arg"
-
 [env.harness]
 max_turns = 1
 
+[env.taskset]
+split = "train"
+
 [env.taskset.scoring.contains_answer]
 weight = 1.0
 ```

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/README.md

@@ -99,6 +99,10 @@ Environments built with Verifiers are self-contained Python modules. To initiali
 ```bash
 prime env init my-env # creates a new template in ./environments/my_env
 ```
+Add an explicit harness loader when the environment owns harness behavior:
+```bash
+prime env init my-env --with-harness
+```
 For OpenEnv integration, use:
 ```bash
 prime env init my-openenv --openenv
@@ -116,7 +120,9 @@ environments/my_env/
 └── README.md           # Documentation
 ```
 
-Environment modules should expose a `load_environment` function which returns an instance of the Environment object, and which can accept custom arguments. For example: 
+Environment modules should expose a `load_environment` function which returns an
+environment object. For simple legacy environments, this can still be a direct
+constructor:
 ```python
 # my_env.py
 import verifiers as vf
@@ -148,7 +154,7 @@ def source():
 async def contains_answer(task, state) -> float:
     return float(task["answer"] in str(state.get("completion") or ""))
 
-def load_taskset(config: vf.TasksetConfig | None = None):
+def load_taskset(config: vf.TasksetConfig):
     return vf.Taskset(source=source, rewards=[contains_answer], config=config)
 
 def load_environment(config: vf.EnvConfig) -> vf.Env:
@@ -169,8 +175,8 @@ env = vf.Env(
 ```
 
 The same environment package is the unit used by evals and `prime-rl`. The
-trainer owns model, endpoint, sampling, and rollout count; v1-specific taskset
-and harness options stay under `env.taskset` and `env.harness`:
+trainer owns model, endpoint, sampling, and rollout count; v1-specific options
+stay on the taskset or harness config that owns them:
 
 ```toml
 # configs/rl/my-v1-env.toml
@@ -185,12 +191,12 @@ max_tokens = 4096
 [[env]]
 id = "my-env"
 
-[env.args]
-arg1 = "non-th-arg"
-
 [env.harness]
 max_turns = 1
 
+[env.taskset]
+split = "train"
+
 [env.taskset.scoring.contains_answer]
 weight = 1.0
 ```

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_eval_cli.py

@@ -13,6 +13,7 @@ import verifiers.scripts.eval as vf_eval
 import verifiers.utils.eval_utils
 from verifiers.types import GenerateOutputs
 from verifiers.utils.eval_utils import load_toml_config
+from verifiers.utils.path_utils import get_eval_results_path
 from verifiers.utils.save_utils import states_to_outputs
 
 
@@ -706,6 +707,34 @@ def test_load_toml_config_multi_env():
         assert result[1]["env_id"] == "env2"
 
 
+def test_load_toml_config_duplicate_envs_accept_names():
+    """Duplicate env ids can be labeled and configured independently."""
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            '[[eval]]\nid = "env1"\nname = "env1-short"\n'
+            "[eval.args]\n"
+            'split = "short"\n\n'
+            '[[eval]]\nid = "env1"\nname = "env1-long"\n'
+            "[eval.args]\n"
+            'split = "long"\n'
+        )
+        f.flush()
+        result = load_toml_config(Path(f.name))
+
+    assert len(result) == 2
+    assert [config["env_id"] for config in result] == ["env1", "env1"]
+    assert [config["name"] for config in result] == ["env1-short", "env1-long"]
+    assert [config["env_args"]["split"] for config in result] == ["short", "long"]
+
+
+def test_load_toml_config_rejects_global_name():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write('name = "shared-name"\n\n[[eval]]\nid = "env1"\n')
+        f.flush()
+        with pytest.raises(ValueError, match="Invalid global field"):
+            load_toml_config(Path(f.name))
+
+
 def test_load_toml_config_with_env_args():
     """Multiple sections with env_args field loads correctly."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
@@ -815,6 +844,28 @@ def test_cli_multi_env_via_toml_config(monkeypatch, run_cli):
     assert configs[1].env_id == "env2"
 
 
+def test_cli_duplicate_env_names_disambiguate_result_paths(monkeypatch, run_cli):
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            '[[eval]]\nid = "env1"\nname = "env1-short"\n'
+            "[eval.args]\n"
+            'split = "short"\n\n'
+            '[[eval]]\nid = "env1"\nname = "env1-long"\n'
+            "[eval.args]\n"
+            'split = "long"\n'
+        )
+        f.flush()
+        captured = run_cli(monkeypatch, {"env_id_or_config": f.name})
+
+    configs = captured["configs"]
+    assert len(configs) == 2
+    assert [config.env_id for config in configs] == ["env1", "env1"]
+    assert [config.name for config in configs] == ["env1-short", "env1-long"]
+    assert [config.env_args["split"] for config in configs] == ["short", "long"]
+    assert get_eval_results_path(configs[0]).parent.name.startswith("env1-short--")
+    assert get_eval_results_path(configs[1]).parent.name.startswith("env1-long--")
+
+
 def test_cli_toml_ignores_cli_args(monkeypatch, run_cli):
     """TOML config ignores CLI args, uses defaults for unspecified values."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_eval_display.py

@@ -11,9 +11,11 @@ def make_config(
     independent_scoring: bool = False,
     endpoint_id: str | None = None,
     client_config: ClientConfig | None = None,
+    name: str | None = None,
 ) -> EvalConfig:
     return EvalConfig(
         env_id="dummy-env",
+        name=name,
         env_args={},
         env_dir_path="./environments",
         endpoint_id=endpoint_id,
@@ -82,6 +84,20 @@ def test_format_client_target_uses_single_resolved_base_url() -> None:
     assert EvalDisplay._format_client_target(config) == "http://localhost:8001/v1"
 
 
+def test_display_uses_eval_name_for_duplicate_env_labels() -> None:
+    display = EvalDisplay(
+        [
+            make_config(max_concurrent=1, name="dummy-env-short"),
+            make_config(max_concurrent=1, name="dummy-env-long"),
+        ]
+    )
+
+    rendered = render_plain(display._make_compact_env_row(0))
+
+    assert "dummy-env-short" in rendered
+    assert "dummy-env-long" not in rendered
+
+
 def render_plain(renderable) -> str:
     console = Console(width=100, record=True)
     console.print(renderable)

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_eval_utils.py

@@ -87,6 +87,22 @@ def test_print_results_single_rollout(capsys, make_metadata, make_state, make_in
     assert "r1: [0.1, 0.2, 0.3]" in captured.out
 
 
+def test_print_results_includes_eval_name(capsys, make_metadata, make_output):
+    from verifiers.utils.eval_utils import print_results
+
+    metadata = make_metadata(env_id="env1")
+    metadata["name"] = "env1-short"
+    results = GenerateOutputs(
+        outputs=[make_output(example_id=0, reward=1.0)],
+        metadata=metadata,
+    )
+
+    print_results(results)
+    captured = capsys.readouterr()
+
+    assert "Environment: env1-short (env1)" in captured.out
+
+
 def test_print_results_three_rollouts(capsys, make_metadata, make_state, make_input):
     """Test print_results with three rollouts per example."""
     from verifiers.utils.eval_utils import print_results

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_langchain_deep_agents_wikispeedia.py

@@ -57,7 +57,7 @@ def test_wikispeedia_loads_as_v1_taskset_harness(
 ) -> None:
     module = load_module(monkeypatch)
 
-    env = module.load_environment(config=vf.EnvConfig(), train_size=1, eval_size=1)
+    env = module.load_environment(config=module.WikispeediaEnvConfig())
 
     assert isinstance(env, vf.Env)
     assert isinstance(env.taskset, vf.Taskset)
@@ -65,6 +65,43 @@ def test_wikispeedia_loads_as_v1_taskset_harness(
     assert env.taskset.taskset_id == "langchain-deep-agents-wikispeedia"
 
 
+def test_wikispeedia_env_config_reaches_taskset_and_harness(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module = load_module(monkeypatch)
+    wiki = make_small_wiki(module)
+    monkeypatch.setattr(module, "load_wiki_graph", lambda cache_dir=None: wiki)
+
+    env = module.load_environment(
+        config=module.WikispeediaEnvConfig(
+            taskset={
+                "train_size": 2,
+                "eval_size": 1,
+                "min_path_length": 1,
+                "max_path_length": 1,
+                "eval_target_fraction": 0.5,
+                "allow_go_back": False,
+                "links_only": True,
+                "max_turns": 7,
+            },
+            harness={
+                "max_turns": 8,
+                "timeout_seconds": 9.0,
+            },
+        )
+    )
+
+    train_rows = list(env.taskset.source())
+    eval_rows = list(env.taskset.eval_source())
+
+    assert len(train_rows) == 2
+    assert len(eval_rows) == 1
+    assert train_rows[0]["max_turns"] == 7
+    assert env.harness.config.max_turns == 8
+    assert env.harness.config.timeout_seconds == 9.0
+    assert [tool.__name__ for tool in env.taskset.toolsets[0].tools] == ["click_link"]
+
+
 def test_wikispeedia_rows_use_v1_task_shape(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
@@ -90,11 +127,13 @@ def test_wikispeedia_taskset_sources_use_disjoint_target_split(
     wiki = make_small_wiki(module)
     monkeypatch.setattr(module, "load_wiki_graph", lambda cache_dir=None: wiki)
     taskset = module.load_taskset(
-        train_size=2,
-        eval_size=1,
-        min_path_length=1,
-        max_path_length=1,
-        eval_target_fraction=0.5,
+        config=module.WikispeediaTasksetConfig(
+            train_size=2,
+            eval_size=1,
+            min_path_length=1,
+            max_path_length=1,
+            eval_target_fraction=0.5,
+        )
     )
 
     train_rows = list(taskset.source())
@@ -114,8 +153,12 @@ def test_wikispeedia_efficiency_weight_uses_fresh_reward_wrapper(
     wiki = make_small_wiki(module)
     monkeypatch.setattr(module, "load_wiki_graph", lambda cache_dir=None: wiki)
 
-    weighted = module.load_taskset(efficiency_weight=0.5)
-    plain = module.load_taskset(efficiency_weight=0.0)
+    weighted = module.load_taskset(
+        config=module.WikispeediaTasksetConfig(efficiency_weight=0.5)
+    )
+    plain = module.load_taskset(
+        config=module.WikispeediaTasksetConfig(efficiency_weight=0.0)
+    )
 
     assert any(fn.__name__ == "path_efficiency" for fn in weighted.rewards)
     assert any(fn is module.path_efficiency for fn in plain.metrics)
@@ -127,13 +170,17 @@ def test_wikispeedia_taskset_owns_navigation_tools(
 ) -> None:
     module = load_module(monkeypatch)
 
-    taskset = module.load_taskset(allow_go_back=True)
+    taskset = module.load_taskset(
+        config=module.WikispeediaTasksetConfig(allow_go_back=True)
+    )
     names = [tool.__name__ for tool in taskset.toolsets[0].tools]
-    no_back = module.load_taskset(allow_go_back=False)
+    no_back = module.load_taskset(
+        config=module.WikispeediaTasksetConfig(allow_go_back=False)
+    )
 
     assert names == ["click_link", "go_back"]
     assert [tool.__name__ for tool in no_back.toolsets[0].tools] == ["click_link"]
-    assert module.load_harness().toolsets == []
+    assert module.load_harness(config=module.WikispeediaHarnessConfig()).toolsets == []
 
 
 def test_wikispeedia_system_prompt_matches_available_tools(
@@ -141,8 +188,12 @@ def test_wikispeedia_system_prompt_matches_available_tools(
 ) -> None:
     module = load_module(monkeypatch)
 
-    with_back = module.load_taskset(allow_go_back=True)
-    without_back = module.load_taskset(allow_go_back=False)
+    with_back = module.load_taskset(
+        config=module.WikispeediaTasksetConfig(allow_go_back=True)
+    )
+    without_back = module.load_taskset(
+        config=module.WikispeediaTasksetConfig(allow_go_back=False)
+    )
 
     assert "go_back" in with_back.system_prompt[0]["content"]
     assert "go_back" not in without_back.system_prompt[0]["content"]
@@ -156,12 +207,16 @@ async def test_wikispeedia_tools_resolve_through_v1_runtime(
     module = load_module(monkeypatch)
     wiki = make_small_wiki(module)
     monkeypatch.setattr(module, "load_wiki_graph", lambda cache_dir=None: wiki)
-    env = module.load_environment(
-        config=vf.EnvConfig(),
-        train_size=2,
-        eval_size=1,
-        min_path_length=1,
-        max_path_length=1,
+    env = vf.Env(
+        taskset=module.load_taskset(
+            config=module.WikispeediaTasksetConfig(
+                train_size=2,
+                eval_size=1,
+                min_path_length=1,
+                max_path_length=1,
+            )
+        ),
+        harness=module.load_harness(config=module.WikispeediaHarnessConfig()),
     )
     task = module.vf.Task(list(env.taskset.source())[0]).freeze()
     state = module.vf.State.for_task(task)

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_mcp_search_env.py

@@ -26,7 +26,9 @@ def _load_mcp_search_module() -> Any:
 def test_mcp_search_env_is_v1_only() -> None:
     module = _load_mcp_search_module()
 
-    env = module.load_environment(config=vf.EnvConfig(), max_turns=4)
+    env = module.load_environment(
+        config=module.MCPSearchEnvConfig(taskset={"max_turns": 4})
+    )
 
     assert isinstance(env, vf.Env)
     assert isinstance(env.taskset, vf.Taskset)
@@ -40,7 +42,7 @@ def test_mcp_search_env_is_v1_only() -> None:
 def test_mcp_search_default_taskset_has_stable_non_doc_fixture() -> None:
     module = _load_mcp_search_module()
 
-    rows = module.load_taskset().rows()
+    rows = module.load_taskset(config=module.MCPSearchTasksetConfig()).rows()
 
     assert len(rows) >= 10
     assert len({row["answer"] for row in rows}) == len(rows)
@@ -52,7 +54,7 @@ def test_mcp_search_taskset_accepts_v1_taskset_config() -> None:
     module = _load_mcp_search_module()
 
     env = module.load_environment(
-        config=vf.EnvConfig(taskset={"max_turns": 3}),
+        config=module.MCPSearchEnvConfig(taskset={"max_turns": 3}),
     )
     rows = env.taskset.rows()
 

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_opencode_harbor.py

@@ -28,7 +28,7 @@ def _load_opencode_module() -> Any:
 def test_load_environment_uses_v1_taskset_and_harness() -> None:
     module = _load_opencode_module()
 
-    env = module.load_environment(config=vf.EnvConfig())
+    env = module.load_environment(config=module.OpenCodeHarborEnvConfig())
 
     assert isinstance(env, vf.Env)
     assert isinstance(env.taskset, vf.HarborTaskset)
@@ -52,7 +52,7 @@ def test_load_environment_accepts_v1_taskset_and_harness_config() -> None:
     module = _load_opencode_module()
 
     env = module.load_environment(
-        config=vf.EnvConfig(
+        config=module.OpenCodeHarborEnvConfig(
             taskset={
                 "task_names": ["task-a"],
                 "cpu_cores": 1.5,

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_path_utils.py

@@ -3,6 +3,7 @@ from pathlib import Path
 
 from verifiers.utils.path_utils import (
     find_latest_incomplete_eval_results_path,
+    get_eval_runs_dir,
     is_valid_eval_results_path,
 )
 
@@ -69,6 +70,19 @@ def test_find_latest_incomplete_eval_results_path_returns_none_when_no_match(
     assert result is None
 
 
+def test_get_eval_runs_dir_uses_name_as_result_label(tmp_path: Path):
+    runs_dir = get_eval_runs_dir(
+        env_id="dummy-env",
+        name="dummy-env-short",
+        model="openai/gpt-4.1-mini",
+        output_dir=str(tmp_path / "outputs"),
+    )
+
+    assert runs_dir == (
+        tmp_path / "outputs" / "evals" / "dummy-env-short--openai--gpt-4.1-mini"
+    )
+
+
 def test_is_valid_eval_results_path_requires_files(tmp_path: Path):
     run_dir = tmp_path / "run"
     run_dir.mkdir()

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_save_utils.py

@@ -32,6 +32,7 @@ from verifiers.utils.save_utils import (
     make_serializable,
     save_new_outputs,
     states_to_outputs,
+    truncate_malformed_trailing_line,
     validate_resume_metadata,
 )
 from verifiers.utils.usage_utils import StateUsageTracker, response_usage_tokens
@@ -488,6 +489,9 @@ class TestSaveNewOutputs:
             "\n".join(lines + [malformed_trailing_line]), encoding="utf-8"
         )
 
+        # Caller drops the partial trailing row before appending so the new
+        # row lands on a valid JSONL boundary.
+        truncate_malformed_trailing_line(outputs_path)
         save_new_outputs(
             [{"example_id": 3, "label": "row-3"}],
             results_path,

{verifiers-0.1.15.dev5 → verifiers-0.1.15.dev7}/tests/test_v1_bfcl.py

@@ -75,12 +75,12 @@ def test_bfcl_public_loader_is_v1_only(monkeypatch: pytest.MonkeyPatch) -> None:
     seen_taskset_config: vf.TasksetConfig | None = None
     seen_harness_config: vf.HarnessConfig | None = None
 
-    def fake_taskset(config: vf.TasksetConfig | None = None) -> vf.Taskset:
+    def fake_taskset(config: vf.TasksetConfig) -> vf.Taskset:
         nonlocal seen_taskset_config
         seen_taskset_config = config
         return vf.Taskset(source=[], config=config)
 
-    def fake_harness(config: vf.HarnessConfig | None = None) -> vf.Harness:
+    def fake_harness(config: vf.HarnessConfig) -> vf.Harness:
         nonlocal seen_harness_config
         seen_harness_config = config
         return vf.Harness(config=config)
@@ -89,9 +89,13 @@ def test_bfcl_public_loader_is_v1_only(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setattr(bfcl, "load_harness", fake_harness)
 
     env = bfcl.load_environment(
-        config=vf.EnvConfig(),
-        test_category="simple_python",
-        examples_per_category=0,
+        config=bfcl.BFCLEnvConfig(
+            taskset=bfcl.BFCLTasksetConfig(
+                test_category="simple_python",
+                examples_per_category=0,
+            ),
+            harness=bfcl.BFCLHarnessConfig(),
+        )
     )
 
     assert isinstance(env, vf.Env)
@@ -110,12 +114,12 @@ def test_bfcl_loader_supports_category_groups(
     seen_taskset_categories = []
     seen_harness_categories = []
 
-    def fake_taskset(config: vf.TasksetConfig | None = None) -> vf.Taskset:
+    def fake_taskset(config: vf.TasksetConfig) -> vf.Taskset:
         assert isinstance(config, bfcl.BFCLTasksetConfig)
         seen_taskset_categories.append(config.test_category)
         return vf.Taskset(source=[{"question": "q", "answer": "a"}], config=config)
 
-    def fake_harness(config: vf.HarnessConfig | None = None) -> vf.Harness:
+    def fake_harness(config: vf.HarnessConfig) -> vf.Harness:
         assert isinstance(config, bfcl.BFCLHarnessConfig)
         seen_harness_categories.append(config.test_category)
         return vf.Harness(config=config)
@@ -124,9 +128,13 @@ def test_bfcl_loader_supports_category_groups(
     monkeypatch.setattr(bfcl, "load_harness", fake_harness)
 
     env = bfcl.load_environment(
-        config=vf.EnvConfig(),
-        test_categories=["simple_python", "simple_java"],
-        examples_per_category=0,
+        config=bfcl.BFCLEnvConfig(
+            taskset=bfcl.BFCLTasksetConfig(
+                test_categories=["simple_python", "simple_java"],
+                examples_per_category=0,
+            ),
+            harness=bfcl.BFCLHarnessConfig(),
+        )
     )
 
     assert isinstance(env, root_vf.EnvGroup)