verifiers 0.1.15.dev12__tar.gz → 0.1.15.dev14__tar.gz

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev12
+Version: 0.1.15.dev14
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/tests/test_eval_cli.py

@@ -1,4 +1,3 @@
-import argparse
 import importlib
 import os
 import sys
@@ -11,7 +10,7 @@ import pytest
 
 import verifiers.scripts.eval as vf_eval
 import verifiers.utils.eval_utils
-from verifiers.types import EndpointConfig, GenerateOutputs
+from verifiers.types import EndpointConfig, EvalConfig, GenerateOutputs
 from verifiers.utils.eval_utils import load_toml_config
 from verifiers.utils.path_utils import get_eval_results_path
 from verifiers.utils.save_utils import states_to_outputs
@@ -25,6 +24,24 @@ def endpoint(**values: object) -> EndpointConfig:
     return EndpointConfig.model_validate(values)
 
 
+def test_eval_config_accepts_id_shorthand():
+    config = EvalConfig.model_validate(
+        {
+            "id": "env1",
+            "env_args": {},
+            "env_dir_path": "./environments",
+            "model": "openai/gpt-4.1-mini",
+            "client_config": {},
+            "sampling_args": {},
+            "num_examples": 1,
+            "rollouts_per_example": 1,
+            "max_concurrent": 1,
+        }
+    )
+
+    assert config.env_id == "env1"
+
+
 @pytest.fixture
 def run_cli(make_metadata, make_state, make_input):
     def _run_cli(
@@ -69,6 +86,7 @@ def run_cli(make_metadata, make_state, make_input):
             "save_to_hf_hub": False,
             "hf_hub_dataset_name": "",
             "extra_env_kwargs": {},
+            "env_config_overrides": [],
             "max_retries": 0,
             "fullscreen": False,
             "disable_tui": False,
@@ -80,11 +98,7 @@ def run_cli(make_metadata, make_state, make_input):
 
         captured: dict = {"sampling_args": None, "configs": []}
 
-        monkeypatch.setattr(
-            argparse.ArgumentParser,
-            "parse_args",
-            lambda self: args_namespace,
-        )
+        monkeypatch.setattr(vf_eval, "parse_args", lambda argv=None: args_namespace)
         monkeypatch.setattr(vf_eval, "setup_logging", lambda *_, **__: None)
         if fail_on_load_endpoints:
             monkeypatch.setattr(vf_eval, "load_endpoints", fail_load_endpoints)
@@ -135,6 +149,97 @@ def test_cli_single_env_id(monkeypatch, run_cli):
     assert configs[0].env_id == "env1"
 
 
+def test_parse_args_accepts_v1_env_config_overrides():
+    args = vf_eval.parse_args(
+        [
+            "env1",
+            "--taskset.id",
+            "my-id",
+            "--harness.id",
+            "my-harness",
+            "--harness.max-turns",
+            "4",
+        ]
+    )
+
+    assert args.env_config_overrides == [
+        "--taskset.id",
+        "my-id",
+        "--harness.id",
+        "my-harness",
+        "--harness.max-turns",
+        "4",
+    ]
+
+
+def test_parse_args_rejects_unknown_eval_flags():
+    with pytest.raises(SystemExit):
+        vf_eval.parse_args(["env1", "--unknown-flag", "value"])
+
+
+def test_cli_v1_env_config_overrides_preserve_env_args_config(
+    tmp_path: Path, monkeypatch, run_cli
+):
+    module_name = f"cli_override_env_{time.time_ns()}"
+    (tmp_path / f"{module_name}.py").write_text(
+        """
+import verifiers as vf
+
+
+class DemoTasksetConfig(vf.TasksetConfig):
+    count: int = 1
+    enabled: bool = True
+
+
+class DemoHarnessConfig(vf.HarnessConfig):
+    label: str = "base"
+
+
+def load_taskset(config: DemoTasksetConfig):
+    raise RuntimeError("not used")
+
+
+def load_harness(config: DemoHarnessConfig):
+    raise RuntimeError("not used")
+
+
+def load_environment(config: vf.EnvConfig):
+    raise RuntimeError("not used")
+""",
+        encoding="utf-8",
+    )
+    monkeypatch.syspath_prepend(str(tmp_path))
+    importlib.invalidate_caches()
+
+    captured = run_cli(
+        monkeypatch,
+        {
+            "env_id_or_config": module_name,
+            "env_args": {"config": {"taskset": {"count": 2}}},
+            "env_config_overrides": [
+                "--taskset.id",
+                "override-id",
+                "--no-taskset.enabled",
+                "--harness.id",
+                "demo-harness",
+                "--harness.max-turns",
+                "4",
+            ],
+        },
+    )
+
+    assert captured["configs"][0].env_args == {
+        "config": {
+            "taskset": {
+                "taskset_id": "override-id",
+                "count": 2,
+                "enabled": False,
+            },
+            "harness": {"harness_id": "demo-harness", "max_turns": 4},
+        }
+    }
+
+
 def test_get_env_eval_defaults_for_package_module(tmp_path: Path, monkeypatch):
     module_name = f"pkg_env_{time.time_ns()}"
     env_id = module_name.replace("_", "-")

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/tests/test_v1_config_extension.py

@@ -2027,6 +2027,28 @@ def test_env_config_tracks_prebuilt_children() -> None:
     assert env.config.harness.max_turns == 3
 
 
+def test_taskset_and_harness_configs_accept_id_shorthand() -> None:
+    class CustomTasksetConfig(TasksetConfig):
+        taskset_id: str | None = "default-taskset"
+
+    class CustomHarnessConfig(HarnessConfig):
+        harness_id: str | None = "default-harness"
+
+    taskset_config = CustomTasksetConfig.model_validate({"id": "taskset-short"})
+    harness_config = CustomHarnessConfig.model_validate({"id": "harness-short"})
+
+    assert taskset_config.taskset_id == "taskset-short"
+    assert harness_config.harness_id == "harness-short"
+    assert explicit_config_data(taskset_config) == {"taskset_id": "taskset-short"}
+    assert explicit_config_data(harness_config) == {"harness_id": "harness-short"}
+
+    taskset = Taskset(config={"id": "taskset-short"})
+    harness = Harness(config={"id": "harness-short"})
+
+    assert taskset.taskset_id == "taskset-short"
+    assert harness.harness_id == "harness-short"
+
+
 def test_env_rejects_taskset_builders() -> None:
     def load_taskset() -> Taskset:
         return Taskset(config=TasksetConfig())

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/tests/test_v1_textarena_taskset.py

@@ -198,6 +198,50 @@ async def test_textarena_user_steps_env_and_stops_when_game_finishes(fake_textar
     assert state["stop_condition"] == "textarena_done"
 
 
+@pytest.mark.asyncio
+async def test_textarena_user_accepts_structured_assistant_content(fake_textarena):
+    _, fake_ta = fake_textarena
+    taskset = textarena.TextArenaTaskset(
+        config=textarena.TextArenaTasksetConfig(
+            game="FakeWordle-v0",
+            answer_state_key="secret_word",
+            num_train_examples=1,
+            num_eval_examples=0,
+        )
+    )
+    task = taskset.to_task(
+        vf.Task(
+            {
+                "example_id": 0,
+                "prompt": [],
+                "answer": "apple",
+                "textarena": {
+                    "game": "FakeWordle-v0",
+                    "answer_state_key": "secret_word",
+                },
+            }
+        )
+    )
+    state = vf.State.for_task(task)
+    state["completion"] = [
+        vf.AssistantMessage(
+            content=[
+                vf.TextContentPart(text="I will guess "),
+                vf.TextContentPart(text="<guess>[apple]</guess>."),
+            ]
+        )
+    ]
+
+    env = vf.Env(taskset=taskset, harness=vf.Harness(config=vf.HarnessConfig()))
+    state = await env.harness.setup_state(task, state)
+    messages = await env.harness.runtime.user_messages(task, state)
+    ta_env = fake_ta.envs[-1]
+
+    assert ta_env.guesses == ["[apple]"]
+    assert messages == [{"role": "user", "content": "Solved."}]
+    assert state["stop_condition"] == "textarena_done"
+
+
 @pytest.mark.asyncio
 async def test_textarena_user_returns_wordle_feedback_for_unfinished_game(
     fake_textarena,

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.15.dev12"
+__version__ = "0.1.15.dev14"
 
 import importlib
 import os

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/verifiers/scripts/eval.py

@@ -10,12 +10,17 @@ os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
 
 import argparse
 import asyncio
+import inspect
 import importlib.util
 import json
 import logging
 from pathlib import Path
 from typing import Any, cast
 
+from pydantic import BaseModel, create_model
+from pydantic_config import ConfigFileError
+from pydantic_config import cli as parse_pydantic_config_cli
+
 from verifiers import setup_logging
 from verifiers.types import (
     ClientConfig,
@@ -34,8 +39,16 @@ from verifiers.utils.eval_utils import (
     run_evaluations,
     run_evaluations_tui,
 )
+from verifiers.utils.env_utils import (
+    env_config_annotation,
+    env_config_child_types,
+    import_env_module,
+    load_env_config,
+)
 from verifiers.utils.import_utils import load_toml
 from verifiers.utils.install_utils import check_hub_env_installed
+from verifiers.v1.env import EnvConfig
+from verifiers.v1.utils.config_utils import explicit_config_data
 
 logger = logging.getLogger(__name__)
 
@@ -46,6 +59,13 @@ DEFAULT_NUM_EXAMPLES = 5
 DEFAULT_ROLLOUTS_PER_EXAMPLE = 3
 DEFAULT_MAX_CONCURRENT = 32
 DEFAULT_CLIENT_TYPE = "openai_chat_completions"
+ENV_CONFIG_OVERRIDE_FLAG_PREFIXES = (
+    "--taskset.",
+    "--harness.",
+    "--no-taskset.",
+    "--no-harness.",
+)
+ENV_CONFIG_OVERRIDE_GROUP_FLAGS = {"--taskset", "--harness"}
 
 # Provider shorthand configs: maps provider name to (base_url, api_key_var[, client_type])
 PROVIDER_CONFIGS: dict[str, dict[str, str]] = {
@@ -240,6 +260,111 @@ def get_env_eval_defaults(env_id: str) -> dict[str, Any]:
     return defaults
 
 
+def is_env_config_override_flag(token: str) -> bool:
+    return token in ENV_CONFIG_OVERRIDE_GROUP_FLAGS or token.startswith(
+        ENV_CONFIG_OVERRIDE_FLAG_PREFIXES
+    )
+
+
+def validate_env_config_override_args(
+    parser: argparse.ArgumentParser,
+    override_args: list[str],
+) -> None:
+    if not override_args:
+        return
+    if not is_env_config_override_flag(override_args[0]):
+        parser.error(f"unrecognized arguments: {' '.join(override_args)}")
+    invalid_flags = [
+        token
+        for token in override_args
+        if token.startswith("--") and not is_env_config_override_flag(token)
+    ]
+    if invalid_flags:
+        parser.error(f"unrecognized arguments: {' '.join(invalid_flags)}")
+
+
+def merge_config_data(
+    base: dict[str, Any],
+    overrides: dict[str, Any],
+) -> dict[str, Any]:
+    merged = dict(base)
+    for key, value in overrides.items():
+        existing = merged.get(key)
+        if isinstance(existing, dict) and isinstance(value, dict):
+            merged[key] = merge_config_data(existing, value)
+        else:
+            merged[key] = value
+    return merged
+
+
+def env_config_cli_type(
+    config_type: type[EnvConfig],
+    default_config: EnvConfig,
+    child_types: dict[str, type[BaseModel]],
+) -> type[EnvConfig]:
+    fields = {
+        field_name: (child_type, getattr(default_config, field_name))
+        for field_name, child_type in child_types.items()
+    }
+    create_env_model = cast(Any, create_model)
+    return cast(
+        type[EnvConfig],
+        create_env_model(
+            f"{config_type.__name__}CliOverrides",
+            __base__=config_type,
+            **fields,
+        ),
+    )
+
+
+def apply_env_config_cli_overrides(
+    env_id: str,
+    env_args: dict[str, Any],
+    override_args: list[str],
+) -> dict[str, Any]:
+    if not override_args:
+        return dict(env_args)
+
+    module = import_env_module(env_id)
+    env_load_func = getattr(module, "load_environment", None)
+    if env_load_func is None:
+        raise ValueError(f"Environment '{env_id}' does not expose load_environment.")
+
+    sig = inspect.signature(env_load_func)
+    config_type = env_config_annotation(env_load_func, sig)
+    if config_type is None:
+        raise ValueError(
+            "Taskset/harness CLI overrides require a v1 loader shaped as "
+            "load_environment(config: vf.EnvConfig)."
+        )
+
+    merged_env_args = dict(env_args)
+    child_types = env_config_child_types(module, config_type)
+    base_config = load_env_config(
+        module,
+        config_type,
+        merged_env_args.get("config", {}),
+        child_types=child_types,
+    )
+    cli_type = env_config_cli_type(config_type, base_config, child_types)
+    try:
+        config = parse_pydantic_config_cli(
+            cli_type,
+            args=override_args,
+            default=base_config,
+        )
+    except ConfigFileError as exc:
+        raise ValueError(f"Invalid taskset/harness override: {exc}") from exc
+
+    base_config_data = explicit_config_data(merged_env_args.get("config", {}))
+    override_config_data = explicit_config_data(config)
+    merged_env_args["config"] = merge_config_data(
+        base_config_data,
+        override_config_data,
+    )
+    return merged_env_args
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -502,8 +627,12 @@ def build_parser() -> argparse.ArgumentParser:
 def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     parser = build_parser()
     if argv is None:
-        return parser.parse_args()
-    return parser.parse_args(argv)
+        args, env_config_overrides = parser.parse_known_args()
+    else:
+        args, env_config_overrides = parser.parse_known_args(argv)
+    validate_env_config_override_args(parser, env_config_overrides)
+    args.env_config_overrides = env_config_overrides
+    return args
 
 
 def main(argv: list[str] | None = None):
@@ -521,6 +650,10 @@ def main(argv: list[str] | None = None):
 
     # Build raw configs: both paths produce list[dict]
     if args.env_id_or_config.endswith(".toml"):
+        if args.env_config_overrides:
+            raise ValueError(
+                "Taskset/harness CLI overrides are only supported with a single environment id, not TOML config files."
+            )
         path = Path(args.env_id_or_config)
         if not path.is_file():
             raise FileNotFoundError(
@@ -800,6 +933,12 @@ def main(argv: list[str] | None = None):
         else:
             raise ValueError(f"Invalid value for --resume: {resume_arg!r}")
 
+        env_args = apply_env_config_cli_overrides(
+            env_id,
+            dict(raw.get("env_args", {})),
+            list(raw.get("env_config_overrides", [])),
+        )
+
         extra_env_kwargs = dict(raw.get("extra_env_kwargs", {}))
         if raw.get("timeout") is not None:
             extra_env_kwargs["timeout_seconds"] = raw["timeout"]
@@ -807,7 +946,7 @@ def main(argv: list[str] | None = None):
         return EvalConfig(
             env_id=env_id,
             name=name,
-            env_args=raw.get("env_args", {}),
+            env_args=env_args,
             env_dir_path=raw.get("env_dir_path", DEFAULT_ENV_DIR_PATH),
             output_dir=raw.get("output_dir"),
             extra_env_kwargs=extra_env_kwargs,

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/verifiers/types.py

@@ -17,7 +17,14 @@ from typing import (
     cast,
 )
 
-from pydantic import BaseModel, ConfigDict, Field, computed_field, field_validator
+from pydantic import (
+    AliasChoices,
+    BaseModel,
+    ConfigDict,
+    Field,
+    computed_field,
+    field_validator,
+)
 
 if TYPE_CHECKING:
     from anthropic.types import RedactedThinkingBlock
@@ -1307,7 +1314,7 @@ class EvalConfig(BaseModel):
     """Pydantic model for evaluation configuration."""
 
     # environment
-    env_id: str
+    env_id: str = Field(validation_alias=AliasChoices("env_id", "id"))
     name: str | None = None
     env_args: dict
     env_dir_path: str

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/verifiers/utils/env_utils.py

@@ -235,28 +235,14 @@ def load_env_config(
     module: ModuleType,
     config_type: type[EnvConfig],
     value: object,
+    *,
+    child_types: Mapping[str, type[BaseModel]] | None = None,
 ) -> EnvConfig:
-    child_types: dict[str, type[BaseModel]] = {}
-    for field_name, factory_name, base_type in (
-        ("taskset", "load_taskset", TasksetConfig),
-        ("harness", "load_harness", HarnessConfig),
-    ):
-        field_type = config_type_from_annotation(
-            config_type.model_fields[field_name].annotation,
-            base_type,
-            f"{config_type.__name__}.{field_name}",
-        )
-        factory_type = factory_config_type(module, factory_name, base_type)
-        if factory_type is not None:
-            if not issubclass(factory_type, field_type):
-                raise TypeError(
-                    f"{module.__name__}.{factory_name} config type "
-                    f"{factory_type.__name__} does not match "
-                    f"{config_type.__name__}.{field_name}: {field_type.__name__}."
-                )
-            child_types[field_name] = factory_type
-        else:
-            child_types[field_name] = field_type
+    resolved_child_types = (
+        env_config_child_types(module, config_type)
+        if child_types is None
+        else child_types
+    )
 
     data: dict[str, object]
     if isinstance(value, config_type):
@@ -271,7 +257,7 @@ def load_env_config(
     else:
         data = dict(explicit_config_data(value))
     defaults: EnvConfig | None = None
-    for field_name, child_type in child_types.items():
+    for field_name, child_type in resolved_child_types.items():
         if field_name not in data:
             defaults = config_type() if defaults is None else defaults
             child = getattr(defaults, field_name)
@@ -284,7 +270,7 @@ def load_env_config(
             raise TypeError(f"config.{field_name} cannot be None.")
         data[field_name] = child_type.model_validate(explicit_config_data(child))
     config = config_type.model_validate(data)
-    for field_name, child_type in child_types.items():
+    for field_name, child_type in resolved_child_types.items():
         child = getattr(config, field_name)
         if not isinstance(child, child_type):
             raise TypeError(
@@ -294,6 +280,34 @@ def load_env_config(
     return config
 
 
+def env_config_child_types(
+    module: ModuleType,
+    config_type: type[EnvConfig],
+) -> dict[str, type[BaseModel]]:
+    child_types: dict[str, type[BaseModel]] = {}
+    for field_name, factory_name, base_type in (
+        ("taskset", "load_taskset", TasksetConfig),
+        ("harness", "load_harness", HarnessConfig),
+    ):
+        field_type = config_type_from_annotation(
+            config_type.model_fields[field_name].annotation,
+            base_type,
+            f"{config_type.__name__}.{field_name}",
+        )
+        factory_type = factory_config_type(module, factory_name, base_type)
+        if factory_type is not None:
+            if not issubclass(factory_type, field_type):
+                raise TypeError(
+                    f"{module.__name__}.{factory_name} config type "
+                    f"{factory_type.__name__} does not match "
+                    f"{config_type.__name__}.{field_name}: {field_type.__name__}."
+                )
+            child_types[field_name] = factory_type
+        else:
+            child_types[field_name] = field_type
+    return child_types
+
+
 def factory_config_type(
     module: ModuleType,
     factory_name: str,

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/verifiers/v1/harness.py

@@ -2,6 +2,8 @@ import asyncio
 from collections.abc import Awaitable, Callable
 from typing import TYPE_CHECKING, Generic, TypeAlias, TypeVar, cast, final
 
+from pydantic import AliasChoices, Field
+
 import verifiers as vf
 from verifiers.clients.client import Client
 from verifiers.errors import Error, OverlongPromptError
@@ -97,6 +99,10 @@ ProgramRunner: TypeAlias = Callable[[Task, State], Awaitable[ProgramResult]]
 
 class HarnessConfig(LifecycleConfig):
     # Core fields configure harness-owned runtime behavior.
+    harness_id: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("harness_id", "id"),
+    )
     program: ProgramConfig = ProgramConfig()
     model: ModelConfig = ModelConfig()
     system_prompt: PromptInput | SystemPromptConfig | None = None
@@ -108,6 +114,14 @@ class HarnessConfig(LifecycleConfig):
     artifacts: ArtifactsConfig = ArtifactsConfig()
     max_turns: int = 10
 
+    @classmethod
+    def __pydantic_init_subclass__(cls, **kwargs: object) -> None:
+        super().__pydantic_init_subclass__(**kwargs)
+        field = cls.model_fields.get("harness_id")
+        if field is not None:
+            field.validation_alias = AliasChoices("harness_id", "id")
+            cls.model_rebuild(force=True)
+
 
 ConfigT = TypeVar("ConfigT", bound=HarnessConfig)
 
@@ -168,6 +182,12 @@ class Harness(RuntimeOwnerMixin[ConfigT], Generic[ConfigT]):
         self.config = cast(ConfigT, coerce_config(config_type, config))
         with config_ref_context(self.config):
             self.initialize_runtime_refresh()
+            resolved_harness_id = self.config.harness_id
+            if resolved_harness_id is not None and not isinstance(
+                resolved_harness_id, str
+            ):
+                raise TypeError("harness_id must be a string.")
+            self.harness_id = resolved_harness_id or type(self).__name__
             self.program_config = self.config.program.resolve()
             system_prompt_value = self.load_system_prompt(self.config)
             self.system_prompt = normalize_system_prompt(

{verifiers-0.1.15.dev12 → verifiers-0.1.15.dev14}/verifiers/v1/taskset.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from typing import Generic, TypeVar, cast, final
 
 from datasets import Dataset
+from pydantic import AliasChoices, Field
 
 from .config import (
     ConfigSource,
@@ -43,13 +44,24 @@ from .types import (
 
 class TasksetConfig(LifecycleConfig):
     # Core fields configure taskset-owned loaders and runtime behavior.
-    taskset_id: str | None = None
+    taskset_id: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("taskset_id", "id"),
+    )
     system_prompt: PromptInput | SystemPromptConfig | None = None
     user: UserConfig | None = None
     bindings: BindingsConfig = BindingsConfig()
     objects: ObjectsConfig = ObjectsConfig()
     artifacts: ArtifactsConfig = ArtifactsConfig()
 
+    @classmethod
+    def __pydantic_init_subclass__(cls, **kwargs: object) -> None:
+        super().__pydantic_init_subclass__(**kwargs)
+        field = cls.model_fields.get("taskset_id")
+        if field is not None:
+            field.validation_alias = AliasChoices("taskset_id", "id")
+            cls.model_rebuild(force=True)
+
 
 ConfigT = TypeVar("ConfigT", bound=TasksetConfig)