verifiers 0.1.15.dev14__tar.gz → 0.1.15.dev16__tar.gz

{verifiers-0.1.15.dev14 → verifiers-0.1.15.dev16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev14
+Version: 0.1.15.dev16
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -234,7 +234,7 @@ import verifiers as vf
 
 
 class MyTasksetConfig(vf.TasksetConfig):
-    split: str = "train"
+    system_prompt: vf.SystemPrompt = "Reverse text exactly."
 
 
 class MyTaskset(vf.Taskset[MyTasksetConfig]):
@@ -247,7 +247,7 @@ class MyTaskset(vf.Taskset[MyTasksetConfig]):
                 "max_turns": 1,
             }
         ]
-        return [row for row in rows if row["split"] == self.config.split]
+        return [row for row in rows if row["split"] == split]
 
     @vf.reward(weight=1.0)
     async def contains_answer(self, task, state) -> float:
@@ -259,14 +259,20 @@ def load_taskset(config: MyTasksetConfig) -> MyTaskset:
 
 
 def load_environment(config: vf.EnvConfig) -> vf.Env:
-    return vf.Env(taskset=vf.load_taskset(config=config.taskset))
+    """Loader pattern for all Taskset/Harness environments."""
+    return vf.Env(
+        taskset=vf.load_taskset(config=config.taskset),
+        harness=vf.load_harness(config=config.harness),
+    )
 ```
-If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
+The child loader annotation defines the taskset config shape; root
+`load_environment` stays typed as `vf.EnvConfig`. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
 Reusable taskset and harness packages live in `tasksets` and `harnesses`.
 Install them with `uv add "verifiers[packages]"`, or with the narrower
-`verifiers[tasksets]` and `verifiers[harnesses]` extras. For example, Harbor
-task directories can run through the bundled OpenCode CLI harness with:
+`verifiers[tasksets]`, `verifiers[harnesses]`, and backend-specific extras. For
+example, Harbor task directories can run through the bundled OpenCode CLI
+harness with:
 
 ```python
 from harnesses import OpenCode, OpenCodeConfig
@@ -299,7 +305,7 @@ id = "my-env"
 max_turns = 1
 
 [env.taskset]
-split = "train"
+system_prompt = "Reverse text exactly."
 
 [env.taskset.scoring.contains_answer]
 weight = 1.0

{verifiers-0.1.15.dev14 → verifiers-0.1.15.dev16}/README.md

@@ -145,7 +145,7 @@ import verifiers as vf
 
 
 class MyTasksetConfig(vf.TasksetConfig):
-    split: str = "train"
+    system_prompt: vf.SystemPrompt = "Reverse text exactly."
 
 
 class MyTaskset(vf.Taskset[MyTasksetConfig]):
@@ -158,7 +158,7 @@ class MyTaskset(vf.Taskset[MyTasksetConfig]):
                 "max_turns": 1,
             }
         ]
-        return [row for row in rows if row["split"] == self.config.split]
+        return [row for row in rows if row["split"] == split]
 
     @vf.reward(weight=1.0)
     async def contains_answer(self, task, state) -> float:
@@ -170,14 +170,20 @@ def load_taskset(config: MyTasksetConfig) -> MyTaskset:
 
 
 def load_environment(config: vf.EnvConfig) -> vf.Env:
-    return vf.Env(taskset=vf.load_taskset(config=config.taskset))
+    """Loader pattern for all Taskset/Harness environments."""
+    return vf.Env(
+        taskset=vf.load_taskset(config=config.taskset),
+        harness=vf.load_harness(config=config.harness),
+    )
 ```
-If no harness is passed, `vf.Env` uses the base endpoint-backed harness. See
+The child loader annotation defines the taskset config shape; root
+`load_environment` stays typed as `vf.EnvConfig`. See
 **[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
 Reusable taskset and harness packages live in `tasksets` and `harnesses`.
 Install them with `uv add "verifiers[packages]"`, or with the narrower
-`verifiers[tasksets]` and `verifiers[harnesses]` extras. For example, Harbor
-task directories can run through the bundled OpenCode CLI harness with:
+`verifiers[tasksets]`, `verifiers[harnesses]`, and backend-specific extras. For
+example, Harbor task directories can run through the bundled OpenCode CLI
+harness with:
 
 ```python
 from harnesses import OpenCode, OpenCodeConfig
@@ -210,7 +216,7 @@ id = "my-env"
 max_turns = 1
 
 [env.taskset]
-split = "train"
+system_prompt = "Reverse text exactly."
 
 [env.taskset.scoring.contains_answer]
 weight = 1.0

{verifiers-0.1.15.dev14 → verifiers-0.1.15.dev16}/tests/test_eval_cli.py

@@ -994,6 +994,29 @@ def test_load_toml_config_with_args_taskset_harness():
     assert "harness" not in result[0]
 
 
+def test_load_toml_config_allows_taskset_id_without_env_id():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            "[[eval]]\n"
+            "[eval.taskset]\n"
+            'id = "tasksets.harbor"\n'
+            "num_examples = 10\n\n"
+            "[eval.harness]\n"
+            'id = "harnesses.opencode"\n'
+            "max_turns = 5\n"
+        )
+        f.flush()
+        result = load_toml_config(Path(f.name))
+
+    assert result[0]["env_id"] == "tasksets.harbor"
+    assert result[0]["env_args"] == {
+        "config": {
+            "taskset": {"id": "tasksets.harbor", "num_examples": 10},
+            "harness": {"id": "harnesses.opencode", "max_turns": 5},
+        },
+    }
+
+
 def test_load_toml_config_missing_env_section():
     """TOML without [[eval]] section raises ValueError."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:

{verifiers-0.1.15.dev14 → verifiers-0.1.15.dev16}/tests/test_init_script.py

@@ -1,6 +1,4 @@
 from pathlib import Path
-
-import pytest
 import verifiers as vf
 from verifiers.scripts.init import init_environment
 
@@ -21,22 +19,27 @@ def test_init_default_writes_v0_stub(tmp_path: Path) -> None:
     assert "EnvTaskset" not in content
 
 
-def test_init_v1_writes_thin_taskset_template(tmp_path: Path) -> None:
+def test_init_v1_writes_taskset_template(tmp_path: Path) -> None:
     init_environment("bar", path=str(tmp_path), v1=True)
     content = read_env_file(tmp_path, "bar")
 
     assert "class BarTasksetConfig(vf.TasksetConfig):" in content
     assert "class BarTaskset(vf.Taskset[BarTasksetConfig]):" in content
+    assert 'system_prompt: vf.SystemPrompt = "Answer exactly."' in content
+    assert '"""Taskset implementation for bar.' in content
     assert 'def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:' in content
     assert (
-        "def load_system_prompt(self, config: BarTasksetConfig) -> vf.SystemPrompt:"
+        '"""Return serializable task records as a list, generator, or Dataset."""'
         in content
     )
+    assert "def load_system_prompt" not in content
     assert "async def correct_answer(self, task: vf.Task, state: vf.State)" in content
     assert "def load_taskset(config: BarTasksetConfig) -> BarTaskset:" in content
+    assert '"""Typed taskset loader used by vf.load_taskset."""' in content
     assert "return BarTaskset(config=config)" in content
     assert "taskset=vf.load_taskset(config=config.taskset)" in content
-    assert "harness=vf.Harness(config=config.harness)" in content
+    assert '"""Loader pattern for all Taskset/Harness environments."""' in content
+    assert "harness=vf.load_harness(config=config.harness)" in content
     assert "class EnvTaskset(" not in content
     assert "_default_" not in content
     assert 'tasks: str = "load_tasks"' not in content
@@ -49,8 +52,12 @@ def test_init_v1_template_loads_with_vf_load_environment(
     init_environment("loadable-v1", path=str(tmp_path), v1=True)
     monkeypatch.syspath_prepend(str(tmp_path / "loadable_v1"))
 
-    with pytest.raises(RuntimeError, match="Load the system prompt"):
-        vf.load_environment("loadable-v1")
+    env = vf.load_environment("loadable-v1")
+
+    dataset = env.get_dataset()
+
+    assert len(dataset) == 1
+    assert dataset[0]["answer"] == "cba"
 
 
 def test_init_v1_with_harness_writes_harness_stub(tmp_path: Path) -> None:
@@ -92,16 +99,14 @@ def test_init_openenv_writes_v1_taskset_template(tmp_path: Path) -> None:
     content = read_env_file(tmp_path, "openenv-sample")
     pyproject = (tmp_path / "openenv_sample" / "pyproject.toml").read_text()
 
-    assert (
-        "from tasksets.openenv import OpenEnvTaskset, OpenEnvTasksetConfig" in content
-    )
+    assert "from tasksets import OpenEnvTaskset, OpenEnvTasksetConfig" in content
     assert (
         "def load_taskset(config: OpenEnvTasksetConfig) -> OpenEnvTaskset:" in content
     )
     assert "taskset=vf.load_taskset(config=config.taskset)" in content
-    assert "harness=vf.Harness(config=config.harness)" in content
+    assert "harness=vf.load_harness(config=config.harness)" in content
     assert "vf.OpenEnvEnv" not in content
-    assert '"tasksets>=0.1.1"' in pyproject
+    assert '"tasksets[openenv]>=0.1.1"' in pyproject
 
 
 def test_init_openenv_multifile_exports_taskset_loader(tmp_path: Path) -> None:

{verifiers-0.1.15.dev14 → verifiers-0.1.15.dev16}/tests/test_v1_config_extension.py

@@ -1670,18 +1670,151 @@ def test_system_prompt_direct_string_can_contain_colon() -> None:
 
 
 @pytest.mark.asyncio
-async def test_harness_rejects_multiple_system_prompt_sources_by_default() -> None:
+async def test_harness_concats_multiple_system_prompt_sources_by_default() -> None:
     taskset = make_taskset(system_prompt="taskset sys")
     harness = make_harness(
         program={"fn": ref("config_program")}, system_prompt="harness sys"
     )
     Env(taskset=taskset, harness=harness)
     task = next(iter(taskset))
+    state = await harness.setup_state(task, State.for_task(task))
+
+    assert state["system_prompt"] == [
+        {"role": "system", "content": "harness sys"},
+        {"role": "system", "content": "taskset sys"},
+    ]
+
+
+@pytest.mark.asyncio
+async def test_task_system_prompt_overrides_taskset_side_at_runtime() -> None:
+    taskset = make_taskset(system_prompt="taskset sys")
+    harness = make_harness(program={"fn": ref("config_program")})
+    Env(taskset=taskset, harness=harness)
+    task = Task(
+        {
+            "prompt": [{"role": "user", "content": "hi"}],
+            "system_prompt": "task sys",
+        }
+    ).freeze()
+    state = await harness.setup_state(task, State.for_task(task))
+
+    assert state["system_prompt"] == [{"role": "system", "content": "task sys"}]
+
+
+@pytest.mark.asyncio
+async def test_task_override_is_resolved_before_harness_concat() -> None:
+    taskset = make_taskset(system_prompt="taskset sys")
+    harness = make_harness(
+        program={"fn": ref("config_program")}, system_prompt="harness sys"
+    )
+    Env(taskset=taskset, harness=harness)
+    task = Task(
+        {
+            "prompt": [{"role": "user", "content": "hi"}],
+            "system_prompt": "task sys",
+        }
+    ).freeze()
+    state = await harness.setup_state(task, State.for_task(task))
 
-    with pytest.raises(ValueError, match="Multiple system_prompt sources"):
+    assert state["system_prompt"] == [
+        {"role": "system", "content": "harness sys"},
+        {"role": "system", "content": "task sys"},
+    ]
+
+
+@pytest.mark.asyncio
+async def test_system_prompt_strategy_can_concat_taskset_side_first() -> None:
+    taskset = make_taskset(system_prompt="taskset sys")
+    harness = make_harness(
+        program={"fn": ref("config_program")},
+        system_prompt="harness sys",
+        system_prompt_strategy="TH",
+    )
+    Env(taskset=taskset, harness=harness)
+    task = next(iter(taskset))
+    state = await harness.setup_state(task, State.for_task(task))
+
+    assert state["system_prompt"] == [
+        {"role": "system", "content": "taskset sys"},
+        {"role": "system", "content": "harness sys"},
+    ]
+
+
+@pytest.mark.asyncio
+async def test_harness_can_reject_multiple_system_prompt_sides() -> None:
+    taskset = make_taskset(system_prompt="taskset sys")
+    harness = make_harness(
+        program={"fn": ref("config_program")},
+        system_prompt="harness sys",
+        system_prompt_strategy="REJECT",
+    )
+    Env(taskset=taskset, harness=harness)
+    task = next(iter(taskset))
+
+    with pytest.raises(ValueError, match="Multiple system_prompt sides"):
         await harness.setup_state(task, State.for_task(task))
 
 
+@pytest.mark.asyncio
+async def test_system_prompt_side_selection_uses_resolved_taskset_side() -> None:
+    taskset = make_taskset(system_prompt="taskset sys")
+    harness = make_harness(
+        program={"fn": ref("config_program")},
+        system_prompt="harness sys",
+        system_prompt_strategy="T_OR_H",
+    )
+    Env(taskset=taskset, harness=harness)
+    task = Task(
+        {
+            "prompt": [{"role": "user", "content": "hi"}],
+            "system_prompt": "task sys",
+        }
+    ).freeze()
+    state = await harness.setup_state(task, State.for_task(task))
+
+    assert state["system_prompt"] == [{"role": "system", "content": "task sys"}]
+
+
+@pytest.mark.asyncio
+async def test_system_prompt_side_selection_can_prefer_harness() -> None:
+    taskset = make_taskset(system_prompt="taskset sys")
+    harness = make_harness(
+        program={"fn": ref("config_program")},
+        system_prompt="harness sys",
+        system_prompt_strategy="H_OR_T",
+    )
+    Env(taskset=taskset, harness=harness)
+    task = next(iter(taskset))
+    state = await harness.setup_state(task, State.for_task(task))
+
+    assert state["system_prompt"] == [{"role": "system", "content": "harness sys"}]
+
+
+@pytest.mark.asyncio
+async def test_system_prompt_strategy_can_select_exact_sides() -> None:
+    taskset = make_taskset(system_prompt="taskset sys")
+    task = Task({"prompt": [{"role": "user", "content": "hi"}]}).freeze()
+
+    harness_t = make_harness(
+        program={"fn": ref("config_program")},
+        system_prompt="harness sys",
+        system_prompt_strategy="T",
+    )
+    Env(taskset=taskset, harness=harness_t)
+    state_t = await harness_t.setup_state(task, State.for_task(task))
+
+    harness_h = make_harness(
+        program={"fn": ref("config_program")},
+        system_prompt="harness sys",
+        system_prompt_strategy="H",
+    )
+    Env(taskset=taskset, harness=harness_h)
+    state_h = await harness_h.setup_state(task, State.for_task(task))
+
+    assert state_t["system_prompt"] == [{"role": "system", "content": "taskset sys"}]
+    assert state_h["system_prompt"] == [{"role": "system", "content": "harness sys"}]
+
+
 @pytest.mark.asyncio
 async def test_task_max_turns_overrides_harness_default() -> None:
     harness = make_harness(max_turns=9)
@@ -2823,6 +2956,114 @@ def load_environment(config: vf.EnvConfig) -> vf.Env:
     assert configured.taskset.get_dataset()[0]["answer"] == "configured:train"
 
 
+def test_load_environment_composes_component_package_without_root_loader(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    module_name = "component_only_taskset_package"
+    module = types.ModuleType(module_name)
+
+    class LocalTasksetConfig(TasksetConfig):
+        answer: str = "configured"
+
+    class LocalTaskset(Taskset[LocalTasksetConfig]):
+        def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+            return [{"prompt": [], "answer": f"{split}:{self.config.answer}"}]
+
+    def load_taskset(config: LocalTasksetConfig) -> LocalTaskset:
+        return LocalTaskset(config=config)
+
+    module.load_taskset = load_taskset
+    monkeypatch.setitem(sys.modules, module_name, module)
+
+    env = vf.load_environment(
+        "component-only-taskset-package",
+        config={
+            "taskset": {"answer": "composed"},
+            "harness": {"max_turns": 3},
+        },
+    )
+
+    assert env.taskset.get_dataset()[0]["answer"] == "train:composed"
+    assert type(env.harness) is Harness
+    assert env.harness.config.max_turns == 3
+
+
+def test_load_environment_delegates_missing_child_loaders_by_config_id(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    env_module = types.ModuleType("thin_env_package")
+    exec(
+        """
+import verifiers as vf
+
+
+def load_environment(config: vf.EnvConfig) -> vf.Env:
+    return vf.Env(
+        taskset=vf.load_taskset(config=config.taskset),
+        harness=vf.load_harness(config=config.harness),
+    )
+""",
+        env_module.__dict__,
+    )
+    taskset_module = types.ModuleType("external_taskset_pkg")
+    exec(
+        """
+import verifiers as vf
+
+
+class ExternalTasksetConfig(vf.TasksetConfig):
+    answer: str = "external"
+
+
+class ExternalTaskset(vf.Taskset[ExternalTasksetConfig]):
+    def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
+        return [{"prompt": [], "answer": f"{split}:{self.config.answer}"}]
+
+
+def load_taskset(config: ExternalTasksetConfig) -> ExternalTaskset:
+    return ExternalTaskset(config=config)
+""",
+        taskset_module.__dict__,
+    )
+    harness_module = types.ModuleType("external_harness_pkg")
+    exec(
+        """
+import verifiers as vf
+
+
+class ExternalHarnessConfig(vf.HarnessConfig):
+    mode: str = "default"
+
+
+class ExternalHarness(vf.Harness[ExternalHarnessConfig]):
+    pass
+
+
+def load_harness(config: ExternalHarnessConfig) -> ExternalHarness:
+    return ExternalHarness(config=config)
+""",
+        harness_module.__dict__,
+    )
+    monkeypatch.setitem(sys.modules, "thin_env_package", env_module)
+    monkeypatch.setitem(
+        sys.modules, "empty_env_package", types.ModuleType("empty_env_package")
+    )
+    monkeypatch.setitem(sys.modules, "external_taskset_pkg", taskset_module)
+    monkeypatch.setitem(sys.modules, "external_harness_pkg", harness_module)
+
+    config = {
+        "taskset": {"id": "external-taskset-pkg", "answer": "delegated"},
+        "harness": {"id": "external-harness-pkg", "mode": "custom"},
+    }
+    for env_id in ("thin-env-package", "empty-env-package"):
+        env = vf.load_environment(env_id, config=config)
+
+        assert env.taskset.get_dataset()[0]["answer"] == "train:delegated"
+        assert type(env.taskset).__name__ == "ExternalTaskset"
+        assert type(env.harness).__name__ == "ExternalHarness"
+        assert env.harness.config.mode == "custom"
+
+
 def test_load_environment_coerces_base_env_config_with_factory_annotations(
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:

{verifiers-0.1.15.dev14 → verifiers-0.1.15.dev16}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev14"
+__version__ = "0.1.15.dev16"
 
 import importlib
 import os
@@ -95,6 +95,7 @@ __all__ = [
     "SandboxConfig",
     "SystemPrompt",
     "SystemPromptConfig",
+    "SystemPromptStrategy",
     "Toolset",
     "ToolLike",
     "ToolsetConfig",
@@ -232,6 +233,7 @@ _LAZY_IMPORTS = {
     "SignalConfig": "verifiers.v1:SignalConfig",
     "SystemPrompt": "verifiers.v1:SystemPrompt",
     "SystemPromptConfig": "verifiers.v1:SystemPromptConfig",
+    "SystemPromptStrategy": "verifiers.v1:SystemPromptStrategy",
     "ToolLike": "verifiers.v1:ToolLike",
     "Toolset": "verifiers.v1:Toolset",
     "ToolsetConfig": "verifiers.v1:ToolsetConfig",
@@ -343,6 +345,7 @@ if TYPE_CHECKING:
         SignalConfig,
         SystemPrompt,
         SystemPromptConfig,
+        SystemPromptStrategy,
         Task,
         Tasks,
         Taskset,

{verifiers-0.1.15.dev14 → verifiers-0.1.15.dev16}/verifiers/scripts/eval.py

@@ -327,11 +327,12 @@ def apply_env_config_cli_overrides(
 
     module = import_env_module(env_id)
     env_load_func = getattr(module, "load_environment", None)
+    config_type: type[EnvConfig] | None
     if env_load_func is None:
-        raise ValueError(f"Environment '{env_id}' does not expose load_environment.")
-
-    sig = inspect.signature(env_load_func)
-    config_type = env_config_annotation(env_load_func, sig)
+        config_type = EnvConfig
+    else:
+        sig = inspect.signature(env_load_func)
+        config_type = env_config_annotation(env_load_func, sig)
     if config_type is None:
         raise ValueError(
             "Taskset/harness CLI overrides require a v1 loader shaped as "
@@ -339,7 +340,8 @@ def apply_env_config_cli_overrides(
         )
 
     merged_env_args = dict(env_args)
-    child_types = env_config_child_types(module, config_type)
+    base_config_data = explicit_config_data(merged_env_args.get("config", {}))
+    child_types = env_config_child_types(module, config_type, base_config_data)
     base_config = load_env_config(
         module,
         config_type,
@@ -356,7 +358,6 @@ def apply_env_config_cli_overrides(
     except ConfigFileError as exc:
         raise ValueError(f"Invalid taskset/harness override: {exc}") from exc
 
-    base_config_data = explicit_config_data(merged_env_args.get("config", {}))
     override_config_data = explicit_config_data(config)
     merged_env_args["config"] = merge_config_data(
         base_config_data,