verifiers 0.1.15.dev6__tar.gz → 0.1.15.dev8__tar.gz

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev6
+Version: 0.1.15.dev8
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -35,6 +35,7 @@ Requires-Dist: nest-asyncio>=1.6.0
 Requires-Dist: numpy
 Requires-Dist: openai-agents>=0.0.7
 Requires-Dist: openai>=1.108.1
+Requires-Dist: prime-pydantic-config[toml]
 Requires-Dist: prime-sandboxes>=0.2.25
 Requires-Dist: prime-tunnel>=0.1.6
 Requires-Dist: pydantic>=2.11.9
@@ -54,7 +55,7 @@ Requires-Dist: stagehand>=3.0.0; extra == 'browser'
 Provides-Extra: openenv
 Requires-Dist: openenv-core>=0.3.0; extra == 'openenv'
 Provides-Extra: renderers
-Requires-Dist: renderers>=0.1.8.dev0; extra == 'renderers'
+Requires-Dist: renderers>=0.1.8.dev4; extra == 'renderers'
 Provides-Extra: rg
 Requires-Dist: reasoning-gym; extra == 'rg'
 Provides-Extra: rl
@@ -218,22 +219,35 @@ custom harnesses, use the v1 Taskset/Harness path:
 # my_env.py
 import verifiers as vf
 
-def source():
-    yield {
-        "prompt": [{"role": "user", "content": "Reverse abc."}],
-        "answer": "cba",
-        "max_turns": 1,
-    }
-
 @vf.reward(weight=1.0)
 async def contains_answer(task, state) -> float:
     return float(task["answer"] in str(state.get("completion") or ""))
 
-def load_taskset(config: vf.TasksetConfig):
-    return vf.Taskset(source=source, rewards=[contains_answer], config=config)
+class MyTasksetConfig(vf.TasksetConfig):
+    split: str = "train"
+
+
+class MyTaskset(vf.Taskset[MyTasksetConfig]):
+    _default_rewards = (contains_answer,)
+
+    def rows(self) -> list[dict[str, object]]:
+        rows = [
+            {
+                "prompt": [{"role": "user", "content": "Reverse abc."}],
+                "answer": "cba",
+                "split": "train",
+                "max_turns": 1,
+            }
+        ]
+        return [row for row in rows if row["split"] == self.config.split]
+
+
+class MyEnvConfig(vf.EnvConfig):
+    taskset: MyTasksetConfig = MyTasksetConfig()
+
 
-def load_environment(config: vf.EnvConfig) -> vf.Env:
-    return vf.Env(taskset=load_taskset(config=config.taskset))
+def load_environment(config: MyEnvConfig) -> vf.Env:
+    return vf.Env(taskset=MyTaskset(config=config.taskset))
 ```
 If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
@@ -244,8 +258,8 @@ harness with:
 
 ```python
 env = vf.Env(
-    taskset=vf.HarborTaskset(),
-    harness=vf.OpenCode(),
+    taskset=vf.HarborTaskset(config=vf.HarborTasksetConfig()),
+    harness=vf.OpenCode(config=vf.OpenCodeConfig()),
 )
 ```
 
@@ -282,16 +296,6 @@ prime env install my-env
 
 For self-managed training launch commands, use the `prime-rl` documentation.
 
-To install the environment module into your project, do:
-```bash
-prime env install my-env # installs from ./environments/my_env
-```
-
-To install an environment from the Environments Hub into your project, do:
-```bash
-prime env install primeintellect/math-python
-```
-
 To run a local evaluation with any OpenAI-compatible model, do:
 ```bash
 prime eval run my-env -m openai/gpt-5-nano # run and save eval results locally
@@ -300,7 +304,7 @@ Evaluations use [Prime Inference](https://docs.primeintellect.ai/inference/overv
 
 View local evaluation results in the terminal UI:
 ```bash
-prime eval tui
+prime eval view
 ```
 
 To publish the environment to the [Environments Hub](https://app.primeintellect.ai/dashboard/environments?ex_sort=most_stars), do:

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/README.md

@@ -143,22 +143,35 @@ custom harnesses, use the v1 Taskset/Harness path:
 # my_env.py
 import verifiers as vf
 
-def source():
-    yield {
-        "prompt": [{"role": "user", "content": "Reverse abc."}],
-        "answer": "cba",
-        "max_turns": 1,
-    }
-
 @vf.reward(weight=1.0)
 async def contains_answer(task, state) -> float:
     return float(task["answer"] in str(state.get("completion") or ""))
 
-def load_taskset(config: vf.TasksetConfig):
-    return vf.Taskset(source=source, rewards=[contains_answer], config=config)
+class MyTasksetConfig(vf.TasksetConfig):
+    split: str = "train"
+
+
+class MyTaskset(vf.Taskset[MyTasksetConfig]):
+    _default_rewards = (contains_answer,)
+
+    def rows(self) -> list[dict[str, object]]:
+        rows = [
+            {
+                "prompt": [{"role": "user", "content": "Reverse abc."}],
+                "answer": "cba",
+                "split": "train",
+                "max_turns": 1,
+            }
+        ]
+        return [row for row in rows if row["split"] == self.config.split]
+
+
+class MyEnvConfig(vf.EnvConfig):
+    taskset: MyTasksetConfig = MyTasksetConfig()
+
 
-def load_environment(config: vf.EnvConfig) -> vf.Env:
-    return vf.Env(taskset=load_taskset(config=config.taskset))
+def load_environment(config: MyEnvConfig) -> vf.Env:
+    return vf.Env(taskset=MyTaskset(config=config.taskset))
 ```
 If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
@@ -169,8 +182,8 @@ harness with:
 
 ```python
 env = vf.Env(
-    taskset=vf.HarborTaskset(),
-    harness=vf.OpenCode(),
+    taskset=vf.HarborTaskset(config=vf.HarborTasksetConfig()),
+    harness=vf.OpenCode(config=vf.OpenCodeConfig()),
 )
 ```
 
@@ -207,16 +220,6 @@ prime env install my-env
 
 For self-managed training launch commands, use the `prime-rl` documentation.
 
-To install the environment module into your project, do:
-```bash
-prime env install my-env # installs from ./environments/my_env
-```
-
-To install an environment from the Environments Hub into your project, do:
-```bash
-prime env install primeintellect/math-python
-```
-
 To run a local evaluation with any OpenAI-compatible model, do:
 ```bash
 prime eval run my-env -m openai/gpt-5-nano # run and save eval results locally
@@ -225,7 +228,7 @@ Evaluations use [Prime Inference](https://docs.primeintellect.ai/inference/overv
 
 View local evaluation results in the terminal UI:
 ```bash
-prime eval tui
+prime eval view
 ```
 
 To publish the environment to the [Environments Hub](https://app.primeintellect.ai/dashboard/environments?ex_sort=most_stars), do:

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/pyproject.toml

@@ -53,6 +53,7 @@ dependencies = [
     "setproctitle>=1.3.0",
     "regex<2026.4.4",
     "httpx>=0.27.0",
+    "prime-pydantic-config[toml]",
 ]
 
 [dependency-groups]
@@ -72,7 +73,7 @@ dev = [
     "aiohttp>=3.9.0",
     "python-dotenv>=1.0.0",
     "nltk",
-    "renderers>=0.1.8.dev0",
+    "renderers>=0.1.8.dev4",
 ]
 policy = [
     "semgrep>=1.150.0",
@@ -95,7 +96,7 @@ openenv = [
     "openenv-core>=0.3.0",
 ]
 renderers = [
-    "renderers>=0.1.8.dev0",
+    "renderers>=0.1.8.dev4",
 ]
 rl = [
     "torch>=2.8.0,<2.9.0",
@@ -113,6 +114,7 @@ rl = [
 [tool.uv]
 preview = true
 required-version = ">=0.11.1"
+exclude-newer = "7 days"
 conflicts = [
     [
         { extra = "openenv" },
@@ -123,12 +125,12 @@ conflicts = [
 name = "pypi"
 url = "https://pypi.org/simple"
 default = true
-exclude-newer = "7 days"
 
 [tool.uv.exclude-newer-package]
 # PrimeIntellect-published on PyPI (trusted publisher)
 prime-tunnel = false
 prime-sandboxes = false
+prime-pydantic-config = false
 renderers = false
 openenv-core = false
 

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/tests/test_client_auth_errors.py

@@ -130,6 +130,9 @@ class _OverlongOpenAIChatClient:
     def __init__(self, message: str) -> None:
         self.chat = self._Chat(message)
 
+    async def post(self, *args, **kwargs):  # noqa: ANN002, ANN003
+        return await self.chat.completions.create(*args, **kwargs)
+
 
 @pytest.mark.parametrize(
     "error_message",

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/tests/test_envs.py

@@ -101,18 +101,44 @@ def test_alphabet_sort_v1_validates_parameters():
     spec.loader.exec_module(module)
 
     with pytest.raises(ValueError, match="min_turns must be at least 1"):
-        module.load_taskset(min_turns=0)
+        module.AlphabetSortTaskset(config=module.AlphabetSortTasksetConfig(min_turns=0))
     with pytest.raises(
         ValueError, match="min_turns must be less than or equal to max_turns"
     ):
-        module.load_taskset(min_turns=3, max_turns=2)
+        module.AlphabetSortTaskset(
+            config=module.AlphabetSortTasksetConfig(min_turns=3, max_turns=2)
+        )
     with pytest.raises(ValueError, match="min_names_per_turn must be at least 1"):
-        module.load_taskset(min_names_per_turn=0)
+        module.AlphabetSortTaskset(
+            config=module.AlphabetSortTasksetConfig(min_names_per_turn=0)
+        )
     with pytest.raises(
         ValueError,
         match="min_names_per_turn must be less than or equal to max_names_per_turn",
     ):
-        module.load_taskset(min_names_per_turn=3, max_names_per_turn=2)
+        module.AlphabetSortTaskset(
+            config=module.AlphabetSortTasksetConfig(
+                min_names_per_turn=3,
+                max_names_per_turn=2,
+            )
+        )
+
+
+@pytest.mark.parametrize("env_name", ["alphabet_sort", "math_python"])
+def test_v1_wrapper_rejects_unknown_kwargs(env_name: str):
+    module_path = Path("environments") / env_name / f"{env_name}.py"
+    spec = importlib.util.spec_from_file_location(
+        f"{env_name}_wrapper_test", module_path
+    )
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+
+    with pytest.raises(
+        TypeError, match="Unsupported v1 load_environment kwargs: extra"
+    ):
+        module.load_environment(v1=True, extra=True)
 
 
 @pytest.mark.slow
@@ -127,8 +153,12 @@ def test_env(env_dir: Path, tmp_path_factory: pytest.TempPathFactory):
     repo_root = Path(__file__).parent.parent
     cmd = (
         f"cd {tmp_venv_dir} && uv venv --clear && source .venv/bin/activate && "
-        f"uv pip install {repo_root.as_posix()} && "
-        f"uv pip install {env_dir.absolute().as_posix()}"
+        "uv pip install "
+        "--exclude-newer-package prime-pydantic-config=2026-05-20T00:00:00Z "
+        f"{repo_root.as_posix()} && "
+        "uv pip install "
+        "--exclude-newer-package prime-pydantic-config=2026-05-20T00:00:00Z "
+        f"{env_dir.absolute().as_posix()}"
     )
     try:
         process = subprocess.run(

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/tests/test_eval_cli.py

@@ -13,6 +13,7 @@ import verifiers.scripts.eval as vf_eval
 import verifiers.utils.eval_utils
 from verifiers.types import GenerateOutputs
 from verifiers.utils.eval_utils import load_toml_config
+from verifiers.utils.path_utils import get_eval_results_path
 from verifiers.utils.save_utils import states_to_outputs
 
 
@@ -706,6 +707,34 @@ def test_load_toml_config_multi_env():
         assert result[1]["env_id"] == "env2"
 
 
+def test_load_toml_config_duplicate_envs_accept_names():
+    """Duplicate env ids can be labeled and configured independently."""
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            '[[eval]]\nid = "env1"\nname = "env1-short"\n'
+            "[eval.args]\n"
+            'split = "short"\n\n'
+            '[[eval]]\nid = "env1"\nname = "env1-long"\n'
+            "[eval.args]\n"
+            'split = "long"\n'
+        )
+        f.flush()
+        result = load_toml_config(Path(f.name))
+
+    assert len(result) == 2
+    assert [config["env_id"] for config in result] == ["env1", "env1"]
+    assert [config["name"] for config in result] == ["env1-short", "env1-long"]
+    assert [config["env_args"]["split"] for config in result] == ["short", "long"]
+
+
+def test_load_toml_config_rejects_global_name():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write('name = "shared-name"\n\n[[eval]]\nid = "env1"\n')
+        f.flush()
+        with pytest.raises(ValueError, match="Invalid global field"):
+            load_toml_config(Path(f.name))
+
+
 def test_load_toml_config_with_env_args():
     """Multiple sections with env_args field loads correctly."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
@@ -720,6 +749,92 @@ def test_load_toml_config_with_env_args():
         assert result[0]["env_args"]["max_examples"] == 100
 
 
+def test_load_toml_config_sampling_section_mirrors_chat_template_kwargs():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            "[sampling]\n"
+            "max_tokens = 1024\n"
+            'reasoning_effort = "medium"\n'
+            "enable_thinking = false\n\n"
+            "[sampling.extra_body]\n"
+            'custom = "value"\n\n'
+            "[sampling.extra_body.chat_template_kwargs]\n"
+            "clear_thinking = true\n\n"
+            "[[eval]]\n"
+            'env_id = "env1"\n'
+        )
+        f.flush()
+        result = load_toml_config(Path(f.name))
+
+    assert result[0]["sampling_args"] == {
+        "max_tokens": 1024,
+        "reasoning_effort": "medium",
+        "enable_thinking": False,
+        "extra_body": {
+            "custom": "value",
+            "chat_template_kwargs": {
+                "clear_thinking": True,
+                "reasoning_effort": "medium",
+                "enable_thinking": False,
+            },
+        },
+    }
+
+
+def test_load_toml_config_sampling_args_mirrors_chat_template_kwargs():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            "[[eval]]\n"
+            'env_id = "env1"\n'
+            'sampling_args = { max_tokens = 256, reasoning_effort = "high", enable_thinking = true }\n'
+        )
+        f.flush()
+        result = load_toml_config(Path(f.name))
+
+    assert result[0]["sampling_args"] == {
+        "max_tokens": 256,
+        "reasoning_effort": "high",
+        "enable_thinking": True,
+        "extra_body": {
+            "chat_template_kwargs": {
+                "reasoning_effort": "high",
+                "enable_thinking": True,
+            }
+        },
+    }
+
+
+def test_cli_toml_eval_sampling_section_pipes_thinking_args(monkeypatch, run_cli):
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            "[[eval]]\n"
+            'env_id = "env1"\n\n'
+            "[eval.sampling]\n"
+            "max_tokens = 512\n"
+            'reasoning_effort = "low"\n'
+            "enable_thinking = true\n"
+        )
+        f.flush()
+        captured = run_cli(
+            monkeypatch,
+            {
+                "env_id_or_config": f.name,
+            },
+        )
+
+    assert captured["sampling_args"] == {
+        "max_tokens": 512,
+        "reasoning_effort": "low",
+        "enable_thinking": True,
+        "extra_body": {
+            "chat_template_kwargs": {
+                "reasoning_effort": "low",
+                "enable_thinking": True,
+            }
+        },
+    }
+
+
 def test_load_toml_config_with_args_taskset_harness():
     """args/taskset/harness sections normalize into load_environment kwargs."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
@@ -728,8 +843,10 @@ def test_load_toml_config_with_args_taskset_harness():
             "[eval.args]\n"
             'split = "train"\n\n'
             "[eval.taskset]\n"
+            'id = "user/taskset-package"\n'
             "num_examples = 10\n\n"
             "[eval.harness]\n"
+            'id = "user/harness-package"\n'
             "max_turns = 5\n"
         )
         f.flush()
@@ -740,8 +857,8 @@ def test_load_toml_config_with_args_taskset_harness():
     assert result[0]["env_args"] == {
         "split": "train",
         "config": {
-            "taskset": {"num_examples": 10},
-            "harness": {"max_turns": 5},
+            "taskset": {"id": "user/taskset-package", "num_examples": 10},
+            "harness": {"id": "user/harness-package", "max_turns": 5},
         },
     }
     assert "args" not in result[0]
@@ -815,6 +932,28 @@ def test_cli_multi_env_via_toml_config(monkeypatch, run_cli):
     assert configs[1].env_id == "env2"
 
 
+def test_cli_duplicate_env_names_disambiguate_result_paths(monkeypatch, run_cli):
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            '[[eval]]\nid = "env1"\nname = "env1-short"\n'
+            "[eval.args]\n"
+            'split = "short"\n\n'
+            '[[eval]]\nid = "env1"\nname = "env1-long"\n'
+            "[eval.args]\n"
+            'split = "long"\n'
+        )
+        f.flush()
+        captured = run_cli(monkeypatch, {"env_id_or_config": f.name})
+
+    configs = captured["configs"]
+    assert len(configs) == 2
+    assert [config.env_id for config in configs] == ["env1", "env1"]
+    assert [config.name for config in configs] == ["env1-short", "env1-long"]
+    assert [config.env_args["split"] for config in configs] == ["short", "long"]
+    assert get_eval_results_path(configs[0]).parent.name.startswith("env1-short--")
+    assert get_eval_results_path(configs[1]).parent.name.startswith("env1-long--")
+
+
 def test_cli_toml_ignores_cli_args(monkeypatch, run_cli):
     """TOML config ignores CLI args, uses defaults for unspecified values."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
@@ -838,6 +977,16 @@ def test_cli_toml_ignores_cli_args(monkeypatch, run_cli):
         assert config.rollouts_per_example == 3  # DEFAULT_ROLLOUTS_PER_EXAMPLE
         assert config.max_concurrent == 32  # default
         assert config.sampling_args["max_tokens"] is None  # default
+        assert config.save_results is True
+
+
+def test_cli_toml_respects_save_results_false(monkeypatch, run_cli):
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write('[[eval]]\nenv_id = "env1"\nsave_results = false\n')
+        f.flush()
+        captured = run_cli(monkeypatch, {"env_id_or_config": f.name})
+
+    assert captured["configs"][0].save_results is False
 
 
 def test_cli_toml_per_env_num_examples(monkeypatch, run_cli):
@@ -1212,6 +1361,44 @@ def test_ablation_global_defaults_apply():
     assert all(c["num_examples"] == 100 for c in configs)
 
 
+def test_ablation_sampling_sweep_merges_with_global_sampling_defaults():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            "[sampling]\n"
+            "max_tokens = 1024\n"
+            'reasoning_effort = "medium"\n\n'
+            '[[ablation]]\nenv_id = "my-env"\n\n'
+            "[ablation.sweep]\n"
+            "sampling = [{ temperature = 0.0 }, { temperature = 1.0, enable_thinking = false }]\n"
+        )
+        f.flush()
+        configs = load_toml_config(Path(f.name))
+
+    assert len(configs) == 2
+    assert configs[0]["sampling_args"] == {
+        "max_tokens": 1024,
+        "reasoning_effort": "medium",
+        "temperature": 0.0,
+        "extra_body": {
+            "chat_template_kwargs": {
+                "reasoning_effort": "medium",
+            }
+        },
+    }
+    assert configs[1]["sampling_args"] == {
+        "max_tokens": 1024,
+        "reasoning_effort": "medium",
+        "temperature": 1.0,
+        "enable_thinking": False,
+        "extra_body": {
+            "chat_template_kwargs": {
+                "reasoning_effort": "medium",
+                "enable_thinking": False,
+            }
+        },
+    }
+
+
 def test_ablation_endpoint_id_override_removes_global_model():
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
         f.write(

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/tests/test_eval_display.py

@@ -11,9 +11,11 @@ def make_config(
     independent_scoring: bool = False,
     endpoint_id: str | None = None,
     client_config: ClientConfig | None = None,
+    name: str | None = None,
 ) -> EvalConfig:
     return EvalConfig(
         env_id="dummy-env",
+        name=name,
         env_args={},
         env_dir_path="./environments",
         endpoint_id=endpoint_id,
@@ -82,6 +84,20 @@ def test_format_client_target_uses_single_resolved_base_url() -> None:
     assert EvalDisplay._format_client_target(config) == "http://localhost:8001/v1"
 
 
+def test_display_uses_eval_name_for_duplicate_env_labels() -> None:
+    display = EvalDisplay(
+        [
+            make_config(max_concurrent=1, name="dummy-env-short"),
+            make_config(max_concurrent=1, name="dummy-env-long"),
+        ]
+    )
+
+    rendered = render_plain(display._make_compact_env_row(0))
+
+    assert "dummy-env-short" in rendered
+    assert "dummy-env-long" not in rendered
+
+
 def render_plain(renderable) -> str:
     console = Console(width=100, record=True)
     console.print(renderable)

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/tests/test_eval_utils.py

@@ -87,6 +87,22 @@ def test_print_results_single_rollout(capsys, make_metadata, make_state, make_in
     assert "r1: [0.1, 0.2, 0.3]" in captured.out
 
 
+def test_print_results_includes_eval_name(capsys, make_metadata, make_output):
+    from verifiers.utils.eval_utils import print_results
+
+    metadata = make_metadata(env_id="env1")
+    metadata["name"] = "env1-short"
+    results = GenerateOutputs(
+        outputs=[make_output(example_id=0, reward=1.0)],
+        metadata=metadata,
+    )
+
+    print_results(results)
+    captured = capsys.readouterr()
+
+    assert "Environment: env1-short (env1)" in captured.out
+
+
 def test_print_results_three_rollouts(capsys, make_metadata, make_state, make_input):
     """Test print_results with three rollouts per example."""
     from verifiers.utils.eval_utils import print_results

{verifiers-0.1.15.dev6 → verifiers-0.1.15.dev8}/tests/test_mcp_search_env.py

@@ -1,5 +1,6 @@
 import importlib.util
 import inspect
+import sys
 from pathlib import Path
 from typing import Any
 
@@ -19,6 +20,7 @@ def _load_mcp_search_module() -> Any:
     assert spec.loader is not None
 
     module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
     spec.loader.exec_module(module)
     return module
 
@@ -39,10 +41,20 @@ def test_mcp_search_env_is_v1_only() -> None:
     assert env.taskset.config.max_turns == 4
 
 
+def test_mcp_search_env_preserves_harness_config() -> None:
+    module = _load_mcp_search_module()
+
+    env = module.load_environment(
+        config=module.MCPSearchEnvConfig(harness={"max_turns": 7})
+    )
+
+    assert env.harness.config.max_turns == 7
+
+
 def test_mcp_search_default_taskset_has_stable_non_doc_fixture() -> None:
     module = _load_mcp_search_module()
 
-    rows = module.load_taskset(config=module.MCPSearchTasksetConfig()).rows()
+    rows = module.MCPSearchTaskset(config=module.MCPSearchTasksetConfig()).rows()
 
     assert len(rows) >= 10
     assert len({row["answer"] for row in rows}) == len(rows)