verifiers 0.3.1.dev42__tar.gz → 0.3.1.dev44__tar.gz

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.3.1.dev42
+Version: 0.3.1.dev44
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -34,7 +34,7 @@ Requires-Dist: msgpack>=1.1.2
 Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai-agents>=0.8.2
 Requires-Dist: openai>=2.9.0
-Requires-Dist: prime-pydantic-config[toml]>=0.4.2
+Requires-Dist: prime-pydantic-config[toml]>=0.4.3
 Requires-Dist: prime-sandboxes>=0.2.37
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/pyproject.toml

@@ -47,7 +47,7 @@ dependencies = [
     "setproctitle>=1.3.0",
     "httpx>=0.27.0",
     "aiohttp>=3.9.0",
-    "prime-pydantic-config[toml]>=0.4.2",
+    "prime-pydantic-config[toml]>=0.4.3",
     "uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'",
     "loguru>=0.7.0",
     "tomli-w>=1.0.0",

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/tests/v1/test_judges.py

@@ -184,7 +184,7 @@ def test_judges_reject_shared_reward_keys():
 
 
 def test_reward_name_fallback():
-    # name > id's package name > snake-cased class name (code-level judge with neither).
+    # name > id > snake-cased class name (code-level judge with neither).
     class MyQualityJudge(vf.Judge[float]):
         prompt = "{x}"
 
@@ -192,7 +192,7 @@ def test_reward_name_fallback():
         vf.ReferenceJudge().reward_name == "reference"
     )  # class-name fallback (no id set)
     assert (
-        vf.ReferenceJudge(vf.ReferenceJudgeConfig(id="org/my-judge@1.0.0")).reward_name
+        vf.ReferenceJudge(vf.ReferenceJudgeConfig(id="my-judge")).reward_name
         == "my-judge"
     )
     assert vf.ReferenceJudge(vf.ReferenceJudgeConfig(name="gold")).reward_name == "gold"

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/verifiers/v1/cli/dashboard/eval.py

@@ -24,7 +24,6 @@ from verifiers.v1.utils.format import (
     format_override,
     format_time,
 )
-from verifiers.v1.utils.install import env_name
 from verifiers.v1.utils.interrupt import cleaning_up
 
 if TYPE_CHECKING:
@@ -240,7 +239,7 @@ def Overview(config: EvalConfig) -> Table:
     taskset = config.env.taskset
     env_label = taskset.name if taskset.id else "no taskset"
     if config.env.id:
-        env_label = f"{env_name(config.env.id)}+{env_label}"
+        env_label = f"{config.env.id}+{env_label}"
     # One seat story when every seat resolves the same way (the common case); one
     # row per seat when they diverge (a judge on its own harness/runtime).
     runtimes = {role: getattr(config.env, role).runtime.type for role in seats}

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/verifiers/v1/configs/cli/eval.py

@@ -12,7 +12,6 @@ from verifiers.v1.configs.env import EnvConfig
 from verifiers.v1.configs.serve import ServeConfig
 from verifiers.v1.envs.single_agent import SingleAgentEnvConfig
 from verifiers.v1.types import SamplingConfig
-from verifiers.v1.utils.install import env_name
 
 
 def default_run_name(env: EnvConfig, model: str) -> str:
@@ -24,7 +23,7 @@ def default_run_name(env: EnvConfig, model: str) -> str:
     if taskset.id and env.id:
         # Same compounding as `EnvConfig.env_id`: a `best-of-n+gsm8k` run must
         # not share a name with a plain `gsm8k` one.
-        name = f"{env_name(env.id)}+{name}"
+        name = f"{env.id}+{name}"
     # Every seat's resolved harness, distinct, in role order.
     harness = "+".join(dict.fromkeys(h.name for h in env.agent_harnesses().values()))
     slug = (

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/verifiers/v1/configs/harness.py

@@ -9,12 +9,11 @@ from pydantic import ConfigDict, Field, FiniteFloat
 from pydantic_config import BaseConfig
 
 from verifiers.v1.types import ID
-from verifiers.v1.utils.install import env_name
 
 
 class HarnessConfig(BaseConfig):
     id: ID = "bash"
-    """Local package or Hub `org/name[@version]`, set through the seat's
+    """Installed harness package, set through the seat's
     `--env.<role>.harness.id` (`--env.agent.harness.id` on the single-agent env)."""
     env: dict[str, str] = Field(default_factory=dict)
     """Extra program variables; harness-owned variables take precedence."""
@@ -30,7 +29,7 @@ class HarnessConfig(BaseConfig):
 
     @property
     def name(self) -> str:
-        return env_name(self.id)
+        return self.id
 
     @property
     def resolved_env(self) -> dict[str, str]:

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/verifiers/v1/configs/judge.py

@@ -9,7 +9,6 @@ from pydantic import BaseModel, FiniteFloat, SerializeAsAny
 
 from verifiers.v1.clients import BaseClientConfig
 from verifiers.v1.types import ID, SamplingConfig
-from verifiers.v1.utils.install import env_name
 
 
 class JudgeConfig(BaseClientConfig):
@@ -29,7 +28,7 @@ Judges = list[SerializeAsAny[JudgeConfig]]
 
 
 def judge_key(config: JudgeConfig) -> str:
-    return config.name or env_name(config.id)
+    return config.name or config.id
 
 
 def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
@@ -41,7 +40,7 @@ def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
         if not raw.get("id"):
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, a local package, or a hub `org/name` package)"
+                "`rubric`, or an installed package)"
             )
         resolved.append(judge_config_type(raw["id"]).model_validate(raw))
     return resolved
@@ -52,7 +51,7 @@ def check_judges(entries: Sequence[JudgeConfig]) -> None:
         if not entry.id:
             raise ValueError(
                 "each `judges` entry needs an `id` (a judge plugin: `reference`, "
-                "`rubric`, a local package, or a hub `org/name` package)"
+                "`rubric`, or an installed package)"
             )
     keys = [judge_key(entry) for entry in entries]
     if duplicates := {key for key in keys if keys.count(key) > 1}:

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/verifiers/v1/configs/taskset.py

@@ -7,12 +7,11 @@ from pydantic_config import BaseConfig
 
 from verifiers.v1.configs.task import TaskConfig
 from verifiers.v1.types import ID
-from verifiers.v1.utils.install import env_name
 
 
 class TasksetConfig(BaseConfig):
     id: ID = ""
-    """Local package or Hub `org/name[@version]`, set with `--env.taskset.id` (or the
+    """Installed taskset package, set with `--env.taskset.id` (or the
     positional `eval <taskset-id>`)."""
     task: SerializeAsAny[TaskConfig] = TaskConfig()
     """Config passed to each task, under `--env.taskset.task.*`."""
@@ -22,4 +21,4 @@ class TasksetConfig(BaseConfig):
 
     @property
     def name(self) -> str:
-        return env_name(self.id)
+        return self.id

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/verifiers/v1/types.py

@@ -2,7 +2,7 @@ from collections.abc import Iterable
 from dataclasses import dataclass
 from typing import Annotated, Any, Literal
 
-from pydantic import AfterValidator, AliasChoices, BaseModel, ConfigDict, Field
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field
 from renderers.base import MultiModalData
 from typing_extensions import TypedDict
 
@@ -250,13 +250,5 @@ class SamplingConfig(BaseModel):
 Sampling = SamplingConfig
 
 
-def _validate_id(plugin_id: str) -> str:
-    from verifiers.v1.utils.install_utils import is_hub_env, parse_env_id
-
-    if is_hub_env(plugin_id):
-        parse_env_id(plugin_id)  # raises ValueError on a malformed org/name[@version]
-    return plugin_id
-
-
-ID = Annotated[str, AfterValidator(_validate_id)]
-"""Plugin id: `name`, `org/name`, or `org/name@version`."""
+ID = str
+"""Plugin id: the name of an installed package exporting the plugin."""

{verifiers-0.3.1.dev42 → verifiers-0.3.1.dev44}/verifiers/v1/utils/loaders.py

@@ -1,6 +1,5 @@
 """Resolve taskset, harness, judge, hook, and environment plugins."""
 
-import contextvars
 import functools
 import importlib
 import importlib.util
@@ -25,7 +24,6 @@ from verifiers.v1.judge import Judge, judge_config_cls
 from verifiers.v1.task import Task
 from verifiers.v1.taskset import Taskset
 from verifiers.v1.utils.generic import concrete_type, prefix_validation_error
-from verifiers.v1.utils.install import ensure_installed
 
 
 def builtin_harness_ids() -> list[str]:
@@ -36,12 +34,6 @@ def builtin_harness_ids() -> list[str]:
     return sorted(m.name for m in pkgutil.iter_modules(harnesses.__path__))
 
 
-skip_plugin_install: contextvars.ContextVar[bool] = contextvars.ContextVar(
-    "skip_plugin_install", default=False
-)
-"""Skip installing envs from the Hub during config parse (for callers that read the config but never run the env)."""
-
-
 def narrow_plugin_field(
     data: dict,
     field: str,
@@ -66,10 +58,6 @@ def narrow_plugin_field(
             f"{field}.id needs an id, and none was given (got {ident!r}); "
             f"pass the id right after the flag{hint}"
         )
-    if skip_plugin_install.get():
-        # keep the plugin id-only; don't import/install from the Hub
-        data[field] = {"id": ident}
-        return
     try:
         data[field] = resolve(ident).model_validate({**raw, "id": ident})
     except ValidationError as e:
@@ -80,7 +68,7 @@ def narrow_plugin_field(
 
 
 def _import_plugin(plugin_id: str, kind: str, group: str) -> ModuleType:
-    module = ensure_installed(plugin_id)
+    module = plugin_id.replace("-", "_").lower()
     namespaced = f"{group}.{module}"
     target = namespaced if importlib.util.find_spec(namespaced) else module
     try:
@@ -100,8 +88,8 @@ def _import_plugin(plugin_id: str, kind: str, group: str) -> ModuleType:
         raise ModuleNotFoundError(
             f"{kind} {plugin_id!r} not found (tried to import {target!r}). {article} {kind} is a "
             f"package exporting its {kind.capitalize()} subclass via `__all__` — the built-in "
-            f"ones ship with verifiers in the `{group}` package, installed from "
-            f"the Environments Hub (`org/name`), or authored yourself.{hint}"
+            f"ones ship with verifiers in the `{group}` package; any other package "
+            f"must already be installed.{hint}"
         ) from e
 
 
@@ -160,8 +148,7 @@ def judge_class(judge_id: str) -> type[Judge]:
 
 
 def default_harness_id(taskset_id: str) -> str:
-    # In skip mode, don't import the taskset to probe for a bundled harness.
-    if not taskset_id or skip_plugin_install.get():
+    if not taskset_id:
         return "bash"
     try:
         module = import_taskset(taskset_id)

verifiers-0.3.1.dev42/verifiers/v1/utils/install.py

@@ -1,39 +0,0 @@
-"""Environment/plugin ID helpers and on-demand Hub installation."""
-
-import logging
-
-from verifiers.v1.utils.install_utils import (
-    check_hub_env_installed,
-    install_from_hub,
-    is_hub_env,
-    normalize_package_name,
-    parse_env_id,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def env_name(env_id: str) -> str:
-    """The package name — the id with org and version stripped (``org/gsm8k@1.0`` ->
-    ``gsm8k``). Used for logging, display, and output paths."""
-    return parse_env_id(env_id)[1] if is_hub_env(env_id) else env_id
-
-
-def env_module(env_id: str) -> str:
-    """The importable module name — `env_name` normalized (hyphens -> underscores)."""
-    return normalize_package_name(env_name(env_id))
-
-
-def ensure_installed(env_id: str) -> str:
-    """Make `env_id` importable and return its module name.
-
-    For a hub id (``org/name[@version]``) that isn't installed, install it from the
-    Environments Hub — latest, or the pinned version — the same path `prime env install`
-    uses. A local id is assumed already importable."""
-    if is_hub_env(env_id) and not check_hub_env_installed(env_id):
-        logger.info("installing %s from the environments hub", env_id)
-        if not install_from_hub(env_id):
-            raise ModuleNotFoundError(
-                f"could not install {env_id!r} from the environments hub"
-            )
-    return env_module(env_id)

verifiers-0.3.1.dev42/verifiers/v1/utils/install_utils.py

@@ -1,239 +0,0 @@
-# ruff: noqa
-
-import importlib
-import logging
-import subprocess
-import sys
-from importlib.metadata import PackageNotFoundError, version as package_version
-from pathlib import Path
-from typing import Optional
-from urllib.parse import urlparse
-
-import requests
-
-logger = logging.getLogger(__name__)
-
-ENVIRONMENTS_HUB_URL = "https://api.primeintellect.ai/api/v1/environmentshub"
-
-
-def _uv_pip_cmd(subcommand: str, *args: str) -> list[str]:
-    """Run uv pip against the active Python interpreter for this process."""
-    return ["uv", "pip", subcommand, "--python", sys.executable, *args]
-
-
-def normalize_package_name(name: str) -> str:
-    """Normalize package name according to Python packaging standards."""
-    return name.replace("-", "_").lower()
-
-
-def parse_env_id(env_id: str) -> tuple[str, str, Optional[str]]:
-    """Parse environment ID into (owner, name, version).
-
-    Args:
-        env_id: Environment ID like 'owner/name' or 'owner/name@version'
-
-    Returns:
-        Tuple of (owner, name, version). Version is None if not specified.
-
-    Raises:
-        ValueError: If format is invalid
-    """
-    version = None
-    if "@" in env_id:
-        env_id, version = env_id.rsplit("@", 1)
-
-    parts = env_id.split("/")
-    if len(parts) != 2 or not parts[0] or not parts[1]:
-        raise ValueError(
-            f"Invalid environment ID: '{env_id}'. Expected format: 'owner/name' or 'owner/name@version'"
-        )
-
-    return parts[0], parts[1], version
-
-
-def is_hub_env(env_id: str) -> bool:
-    """Check if env_id refers to a Hub environment (has owner/ prefix)."""
-    return "/" in env_id and not env_id.startswith("./") and not env_id.startswith("/")
-
-
-def is_installed(env_name: str, version: Optional[str] = None) -> bool:
-    """Check if an environment package is installed.
-
-    Args:
-        env_name: Environment name (without owner prefix)
-        version: Optional version to check for
-
-    Returns:
-        True if installed (and version matches if specified)
-    """
-    try:
-        installed_version = package_version(normalize_package_name(env_name))
-    except PackageNotFoundError:
-        return False
-    return not version or version == "latest" or installed_version == version
-
-
-def check_hub_env_installed(env_id: str) -> bool:
-    """Check if a Hub environment is installed.
-
-    For Hub environments (owner/name format), checks if the package is installed.
-    For local environments, returns True (assumes available via env_dir).
-
-    Args:
-        env_id: Environment ID (local name or 'owner/name' or 'owner/name@version')
-
-    Returns:
-        True if environment is available (or is local)
-    """
-    if not is_hub_env(env_id):
-        return True
-
-    _, name, version = parse_env_id(env_id)
-    return is_installed(name, version)
-
-
-def install_from_hub(env_id: str) -> bool:
-    """Install an environment from the Environments Hub.
-
-    Args:
-        env_id: Environment ID like 'owner/name' or 'owner/name@version'
-
-    Returns:
-        True if installation succeeded
-    """
-    owner, name, version = parse_env_id(env_id)
-    version = version or "latest"
-
-    logger.info(f"Fetching environment details for {owner}/{name}@{version}...")
-
-    try:
-        url = f"{ENVIRONMENTS_HUB_URL}/{owner}/{name}/@{version}"
-        response = requests.get(url, timeout=30)
-        response.raise_for_status()
-        data = response.json()
-        details = data.get("data", data)
-    except requests.RequestException as e:
-        logger.error(f"Failed to fetch environment details: {e}")
-        return False
-    except ValueError as e:
-        logger.error(f"Invalid response from Environments Hub: {e}")
-        return False
-
-    simple_index_url = details.get("simple_index_url")
-    wheel_url = details.get("wheel_url")
-    if wheel_url:
-        try:
-            parsed_wheel_url = urlparse(wheel_url)
-            if (
-                parsed_wheel_url.scheme not in ("http", "https")
-                or not parsed_wheel_url.netloc
-            ):
-                wheel_url = None
-        except Exception:
-            wheel_url = None
-
-    if not simple_index_url and not wheel_url:
-        if details.get("visibility") == "PRIVATE":
-            logger.error(
-                f"Cannot install private environment '{env_id}'. "
-                "Use 'prime env pull' to download and install locally."
-            )
-        else:
-            logger.error(f"No installation method available for '{env_id}'")
-        return False
-
-    pkg_name = normalize_package_name(name)
-
-    cmd: list[str]
-    if simple_index_url:
-        if version and version != "latest":
-            pkg_spec = f"{pkg_name}=={version}"
-        else:
-            pkg_spec = pkg_name
-        cmd = [
-            *_uv_pip_cmd("install"),
-            "--upgrade",
-            pkg_spec,
-            "--extra-index-url",
-            simple_index_url,
-        ]
-    else:
-        assert wheel_url is not None
-        cmd = [*_uv_pip_cmd("install"), "--upgrade", wheel_url]
-
-    logger.info(f"Installing {env_id}...")
-    logger.debug(f"Command: {' '.join(cmd)}")
-
-    result = subprocess.run(cmd, capture_output=True, text=True)
-
-    if result.returncode != 0:
-        logger.error(f"Installation failed: {result.stderr}")
-        return False
-
-    logger.info(f"Successfully installed {env_id}")
-    importlib.invalidate_caches()
-    return True
-
-
-def install_from_local(env_name: str, env_dir: str = "./environments") -> bool:
-    """Install an environment from a local directory.
-
-    Args:
-        env_name: Environment name
-        env_dir: Path to environments directory
-
-    Returns:
-        True if installation succeeded
-    """
-    env_folder = normalize_package_name(env_name)
-    env_path = Path(env_dir) / env_folder
-
-    if not env_path.exists():
-        logger.error(f"Local environment not found: {env_path}")
-        return False
-
-    logger.info(f"Installing {env_name} from {env_path}...")
-    result = subprocess.run(
-        [*_uv_pip_cmd("install"), "-e", str(env_path)],
-        capture_output=True,
-        text=True,
-    )
-
-    if result.returncode != 0:
-        logger.error(f"Installation failed: {result.stderr}")
-        return False
-
-    logger.info(f"Successfully installed {env_name}")
-    importlib.invalidate_caches()
-    return True
-
-
-def install_from_repo(env_name: str, branch: str = "main") -> bool:
-    """Install an environment from the verifiers GitHub repo.
-
-    Args:
-        env_name: Environment name
-        branch: Git branch to install from
-
-    Returns:
-        True if installation succeeded
-    """
-    env_folder = normalize_package_name(env_name)
-    pkg_name = env_folder.replace("_", "-")
-
-    url = f"git+https://github.com/PrimeIntellect-ai/verifiers.git@{branch}#subdirectory=environments/{env_folder}"
-
-    logger.info(f"Installing {env_name} from verifiers repo ({branch})...")
-    result = subprocess.run(
-        [*_uv_pip_cmd("install"), f"{pkg_name} @ {url}"],
-        capture_output=True,
-        text=True,
-    )
-
-    if result.returncode != 0:
-        logger.error(f"Installation failed: {result.stderr}")
-        return False
-
-    logger.info(f"Successfully installed {env_name}")
-    importlib.invalidate_caches()
-    return True