verifiers 0.1.12.dev5__tar.gz → 0.1.12.dev6__tar.gz

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.12.dev5
+Version: 0.1.12.dev6
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/tests/test_eval_cli.py

@@ -1061,6 +1061,38 @@ def test_ablation_global_defaults_apply():
     assert all(c["num_examples"] == 100 for c in configs)
 
 
+def test_ablation_endpoint_id_override_removes_global_model():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            'model = "gpt-4.1-mini"\n\n'
+            '[[ablation]]\nenv_id = "my-env"\nendpoint_id = "proxy"\n\n'
+            "[ablation.sweep]\n"
+            "temperature = [0.0]\n"
+        )
+        f.flush()
+        configs = load_toml_config(Path(f.name))
+
+    assert len(configs) == 1
+    assert configs[0]["endpoint_id"] == "proxy"
+    assert "model" not in configs[0]
+
+
+def test_ablation_swept_model_override_removes_global_endpoint_id():
+    with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:
+        f.write(
+            'endpoint_id = "proxy"\n\n'
+            '[[ablation]]\nenv_id = "my-env"\n\n'
+            "[ablation.sweep]\n"
+            'model = ["gpt-4.1-mini"]\n'
+        )
+        f.flush()
+        configs = load_toml_config(Path(f.name))
+
+    assert len(configs) == 1
+    assert configs[0]["model"] == "gpt-4.1-mini"
+    assert "endpoint_id" not in configs[0]
+
+
 def test_ablation_with_eval_blocks():
     """Ablation and eval blocks can coexist."""
     with tempfile.NamedTemporaryFile(suffix=".toml", delete=False, mode="w") as f:

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.12.dev5"
+__version__ = "0.1.12.dev6"
 
 import importlib
 import os

verifiers-0.1.12.dev6/verifiers/cli/commands/eval.py

@@ -0,0 +1,21 @@
+"""Evaluation command module for external hosts."""
+
+from verifiers.scripts.eval import (
+    build_extra_headers,
+    build_parser,
+    main,
+    merge_sampling_args,
+    parse_args,
+)
+
+__all__ = [
+    "build_extra_headers",
+    "build_parser",
+    "merge_sampling_args",
+    "parse_args",
+    "main",
+]
+
+
+if __name__ == "__main__":
+    main()

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/envs/experimental/composable/harnesses/rlm.py

@@ -13,11 +13,11 @@ DEFAULT_APPEND_TO_SYSTEM_PROMPT_PATH = "/task/append_to_system_prompt.txt"
 
 
 def build_install_script(rlm_repo_url: str = DEFAULT_RLM_REPO_URL) -> str:
-    return (
-        "set -e; "
-        'command -v uv >/dev/null 2>&1 || { curl -LsSf https://astral.sh/uv/install.sh | sh; source "$HOME/.local/bin/env"; }; '
-        f'uv tool install --python 3.11 "rlm @ git+https://${{GH_TOKEN}}@{rlm_repo_url}"'
+    raw_base = rlm_repo_url.removesuffix(".git").replace(
+        "github.com", "raw.githubusercontent.com"
     )
+    url = f"https://${{GH_TOKEN}}@{raw_base}/main/install.sh"
+    return f"(curl -fsSL {url} || wget -qO- {url}) > /tmp/rlm-install.sh && bash /tmp/rlm-install.sh"
 
 
 def build_run_command(

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py

@@ -4,10 +4,14 @@ import json
 import logging
 import tarfile
 import tempfile
-import tomllib
 from pathlib import Path
 from typing import Any
 
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib
+
 import verifiers as vf
 from verifiers.envs.experimental.composable import SandboxSpec, SandboxTaskSet
 

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/scripts/eval.py

@@ -90,6 +90,58 @@ PROVIDER_CONFIGS: dict[str, dict[str, str]] = {
 DEFAULT_PROVIDER = "prime"
 
 
+def merge_sampling_args(
+    sampling_args: dict[str, Any] | None,
+    *,
+    max_tokens: int | None = None,
+    temperature: float | None = None,
+    prefer_existing_keys: bool = True,
+    include_none_max_tokens: bool = False,
+) -> dict[str, Any]:
+    merged_sampling_args = dict(sampling_args or {})
+
+    if (not prefer_existing_keys or "max_tokens" not in merged_sampling_args) and (
+        include_none_max_tokens or max_tokens is not None
+    ):
+        merged_sampling_args["max_tokens"] = max_tokens
+
+    if temperature is not None and (
+        not prefer_existing_keys or "temperature" not in merged_sampling_args
+    ):
+        merged_sampling_args["temperature"] = temperature
+
+    return merged_sampling_args
+
+
+def build_extra_headers(raw: dict[str, Any]) -> dict[str, str]:
+    eval_headers_table: dict[str, str] = {}
+    raw_headers = raw.get("headers")
+    if raw_headers is not None:
+        eval_headers_table = _validate_extra_headers_value(raw_headers)
+
+    raw_header_values = raw.get("header")
+    if raw_header_values is None:
+        raw_header_values = []
+    if not isinstance(raw_header_values, list):
+        raise ValueError("'header' must be a list of 'Name: Value' strings")
+
+    eval_headers_from_list: dict[str, str] = {}
+    for header_value in raw_header_values:
+        if not isinstance(header_value, str):
+            raise ValueError(
+                f"Each 'header' entry must be a string 'Name: Value', got: {header_value!r}"
+            )
+        if ":" not in header_value:
+            raise ValueError(f"--header must be 'Name: Value', got: {header_value!r}")
+        key, value = header_value.split(":", 1)
+        key, value = key.strip(), value.strip()
+        if not key:
+            raise ValueError("--header name cannot be empty")
+        eval_headers_from_list[key] = value
+
+    return {**eval_headers_table, **eval_headers_from_list}
+
+
 def get_env_eval_defaults(env_id: str) -> dict[str, Any]:
     """Get eval config defaults from the environment module's pyproject.toml.
 
@@ -147,7 +199,7 @@ def get_env_eval_defaults(env_id: str) -> dict[str, Any]:
     return defaults
 
 
-def main():
+def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "env_id_or_config",
@@ -384,7 +436,18 @@ def main():
         default=None,
         help="Heartbeat URL for uptime monitoring",
     )
-    args = parser.parse_args()
+    return parser
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = build_parser()
+    if argv is None:
+        return parser.parse_args()
+    return parser.parse_args(argv)
+
+
+def main(argv: list[str] | None = None):
+    args = parse_args(argv)
 
     if args.debug:  # only set up console logging in debug mode
         setup_logging(get_log_level(args.verbose))
@@ -561,35 +624,14 @@ def main():
             )
 
         # Merge sampling args
-        merged_sampling_args: dict = {}
-        if raw.get("sampling_args") is not None:
-            merged_sampling_args.update(raw["sampling_args"])
-        if "max_tokens" not in merged_sampling_args:
-            merged_sampling_args["max_tokens"] = raw.get("max_tokens")
-        raw_temp = raw.get("temperature")
-        if raw_temp is not None and "temperature" not in merged_sampling_args:
-            merged_sampling_args["temperature"] = raw_temp
+        merged_sampling_args = merge_sampling_args(
+            raw.get("sampling_args"),
+            max_tokens=raw.get("max_tokens"),
+            temperature=raw.get("temperature"),
+            include_none_max_tokens=True,
+        )
         # Build headers: registry < [[eval]] headers table < header list / --header
-        eval_headers_table: dict[str, str] = {}
-        raw_headers = raw.get("headers")
-        if raw_headers is not None:
-            eval_headers_table = _validate_extra_headers_value(raw_headers)
-
-        eval_headers_from_list: dict[str, str] = {}
-        for h in raw.get("header") or []:
-            if not isinstance(h, str):
-                raise ValueError(
-                    f"Each 'header' entry must be a string 'Name: Value', got: {h!r}"
-                )
-            if ":" not in h:
-                raise ValueError(f"--header must be 'Name: Value', got: {h!r}")
-            k, v = h.split(":", 1)
-            k, v = k.strip(), v.strip()
-            if not k:
-                raise ValueError("--header name cannot be empty")
-            eval_headers_from_list[k] = v
-
-        eval_headers_merged = {**eval_headers_table, **eval_headers_from_list}
+        eval_headers_merged = build_extra_headers(raw)
 
         registry_headers_base: dict[str, str] = {}
         if endpoint_group is not None:

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/serve/server/env_router.py

@@ -101,6 +101,7 @@ class EnvRouter:
         log_level: str | None = None,
         log_dir: str | None = None,
         console_logging: bool = True,
+        json_logging: bool = False,
         *,
         num_workers: int = 1,
         worker_heartbeat_timeout: float = 30.0,
@@ -116,6 +117,7 @@ class EnvRouter:
         self.log_level = log_level
         self.log_dir = log_dir
         self.console_logging = console_logging
+        self.json_logging = json_logging
 
         self.num_workers = num_workers
         self.worker_heartbeat_timeout = worker_heartbeat_timeout
@@ -185,6 +187,7 @@ class EnvRouter:
                 self.log_level,
                 self.log_dir,
                 self.console_logging,
+                self.json_logging,
             ),
             kwargs=dict(
                 worker_id=worker_id,

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/serve/server/env_server.py

@@ -70,6 +70,7 @@ class EnvServer(ABC):
             log_level=log_level,
             log_dir=log_dir,
             console_logging=console_logging,
+            json_logging=json_logging,
             num_workers=num_workers,
             worker_heartbeat_timeout=worker_heartbeat_timeout,
             stats_log_interval=stats_log_interval,

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/serve/server/env_worker.py

@@ -60,6 +60,7 @@ class EnvWorker:
         log_level: str | None = None,
         log_dir: str | None = None,
         console_logging: bool = True,
+        json_logging: bool = False,
         *,
         worker_id: int,
         worker_name: str,
@@ -78,6 +79,7 @@ class EnvWorker:
         logger_kwargs: dict[str, Any] = {
             "console_logging": console_logging,
             "file_logging": log_dir is not None,
+            "json_logging": json_logging,
         }
         if log_level is not None:
             logger_kwargs["level"] = log_level

{verifiers-0.1.12.dev5 → verifiers-0.1.12.dev6}/verifiers/utils/eval_utils.py

@@ -330,8 +330,14 @@ def _expand_ablation(ablation: dict, global_defaults: dict) -> list[dict]:
                 f"sweep.env_args — use one or the other"
             )
 
+    explicit_keys = (set(ablation.keys()) - {"sweep"}) | set(sweep.keys())
+
     # Fixed fields: global defaults overridden by ablation-level fields
     fixed = {**global_defaults, **ablation}
+    if "endpoint_id" in explicit_keys and "model" not in explicit_keys:
+        fixed.pop("model", None)
+    if "model" in explicit_keys and "endpoint_id" not in explicit_keys:
+        fixed.pop("endpoint_id", None)
 
     # Expand cartesian product
     keys = [k for k, _ in dimensions]
@@ -351,7 +357,9 @@ def _expand_ablation(ablation: dict, global_defaults: dict) -> list[dict]:
     return expanded
 
 
-def load_toml_config(path: Path) -> list[dict]:
+def load_toml_config(
+    path: Path, extra_valid_fields: set[str] | None = None
+) -> list[dict]:
     """Loads and validates a TOML config file.
 
     Config format supports global defaults at the top level, with per-eval overrides
@@ -454,6 +462,7 @@ def load_toml_config(path: Path) -> list[dict]:
         "save_to_hf_hub",
         "hf_hub_dataset_name",
     }
+    valid_fields |= extra_valid_fields or set()
 
     # validate global fields
     if global_defaults:
@@ -475,6 +484,10 @@ def load_toml_config(path: Path) -> list[dict]:
             )
         # global defaults, then per-eval overrides
         merged = {**global_defaults, **eval_config}
+        if "endpoint_id" in eval_config and "model" not in eval_config:
+            merged.pop("model", None)
+        if "model" in eval_config and "endpoint_id" not in eval_config:
+            merged.pop("endpoint_id", None)
         merged_eval_list.append(merged)
 
     # expand [[ablation]] blocks into eval configs

verifiers-0.1.12.dev5/verifiers/cli/commands/eval.py

@@ -1,7 +0,0 @@
-"""Evaluation command module for external hosts."""
-
-from verifiers.scripts.eval import main
-
-
-if __name__ == "__main__":
-    main()