verifiers 0.4.1.dev3__py3-none-any.whl → 0.4.1.dev5__py3-none-any.whl

verifiers/v1/runtimes/base.py

@@ -147,6 +147,9 @@ class Runtime(ABC):
     use a host `Tunnel` inward plus `expose` outward."""
 
     supports_user: ClassVar[bool] = False
+    supports_expose: ClassVar[bool] = True
+    """Whether `expose` can publish a port inside this runtime to the outside, so a
+    server here is reachable from another runtime."""
     """Whether `with_user` can run this runtime's commands as another existing user."""
 
     scripts_dir: ClassVar[str] = "/tmp/vf-scripts"

verifiers/v1/runtimes/prime.py

@@ -149,6 +149,7 @@ class PrimeProcess(RuntimeProcess):
 
 class PrimeRuntime(Runtime):
     supports_user: ClassVar[bool] = True
+    supports_expose: ClassVar[bool] = False
     is_local: ClassVar[bool] = False
 
     def __init__(self, config: PrimeConfig, name: str | None = None) -> None:

verifiers/v1/utils/loaders.py

@@ -27,11 +27,21 @@ from verifiers.v1.utils.generic import concrete_type, prefix_validation_error
 
 
 def builtin_harness_ids() -> list[str]:
-    """The harness ids that ship with verifiers (the `verifiers.v1.harnesses`
-    subpackages)."""
+    """The harness ids that ship with verifiers: the `verifiers.v1.harnesses`
+    subpackages that export a `Harness`, by their hyphenated id."""
     from verifiers.v1 import harnesses
 
-    return sorted(m.name for m in pkgutil.iter_modules(harnesses.__path__))
+    ids = []
+    for info in pkgutil.iter_modules(harnesses.__path__):
+        if not info.ispkg:
+            continue
+        module = importlib.import_module(f"{harnesses.__name__}.{info.name}")
+        try:
+            _plugin_class(module, Harness, "harness")
+        except (AttributeError, TypeError):
+            continue
+        ids.append(info.name.replace("_", "-"))
+    return sorted(ids)
 
 
 def narrow_plugin_field(

{verifiers-0.4.1.dev3.dist-info → verifiers-0.4.1.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev3
+Version: 0.4.1.dev5
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -91,6 +91,34 @@ uv tool install prime
 - The [docs](docs/) contain short, human-written guides and overviews about the architecture.
 - The [AGENTS.md](AGENTS.md) and [skills](skills/) are for coding agents and go into more details.
 
+## Contributing
+
+We welcome community contributions. We use [issues](https://github.com/PrimeIntellect-ai/verifiers/issues) to track bugs and feature requests. If you hit a bug, a pain point, or have an idea for a feature, please open an issue.
+
+Contributions are welcome via PR. Please follow these guidelines:
+
+1. Install the [pre-commit hooks](#pre-commit-hooks) so your code is formatted and type-checked.
+2. Keep your PR in "Draft" until it is ready for review.
+3. If your PR resolves an issue, link the issue in the PR description.
+4. Run the [tests](#tests) for the areas you touched.
+
+### Pre-commit hooks
+
+```bash
+uv run pre-commit install
+```
+
+### Tests
+
+Unit tests run one component in isolation, with no network, and mirror the `verifiers/v1` layout. Integration tests run real evals: they need `PRIME_API_KEY` and skip without it.
+
+```bash
+uv run pytest tests/unit -n auto                       # unit
+uv run pytest tests/integration -n 8                   # integration
+uv run pytest tests/integration/test_single_turn.py    # one behavior
+uv run pytest tests/integration -m "docker and codex"  # one runtime and harness
+```
+
 ## Citation
 
 Originally created by Will Brown ([@willccbb](https://github.com/willccbb)).

{verifiers-0.4.1.dev3.dist-info → verifiers-0.4.1.dev5.dist-info}/RECORD

@@ -20,7 +20,6 @@ verifiers/v1/acp/runner.py,sha256=sFsg0UgWeDjpjihejUidIV2yTapVj1Ap0cDTaEktjkI,15
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/cli/debug.py,sha256=OgzXCW9HaTNruZil4f70WP5tR6toKliKaVvV98C5qzE,11665
 verifiers/v1/cli/gepa.py,sha256=IyS3YZCCHVydb-OyPeGdtCYsz6OYqXXddXqYq8RLucM,4374
-verifiers/v1/cli/init.py,sha256=k1wawdQBquiiepPrb0enBT-vdKvfSievgs4CiwSdFUc,7601
 verifiers/v1/cli/output.py,sha256=4FRuQ6UIZCigMfUUO2nCFFJKxls3DtWDXswMCVTxjIA,5010
 verifiers/v1/cli/replay.py,sha256=LkeaV8Sxa7kHue9JKHDg0V_N6KXOcjN_r6RmeQVcxfk,10396
 verifiers/v1/cli/resolve.py,sha256=Q1EHM7wWQo0YkwJA898qtZbYLaIkjIVq3Q7kUp2YIxs,4346
@@ -57,7 +56,6 @@ verifiers/v1/configs/cli/__init__.py,sha256=YMDUPdbRlDJ77dhj3QqzmyRhqNgUvuQdpj_b
 verifiers/v1/configs/cli/debug.py,sha256=CcUZ9MEcv0iAL9w2BkTfnZg20yWJ-Vn8P30-e5tuCic,2741
 verifiers/v1/configs/cli/env.py,sha256=qg2LNOasAU0yg3b0xqXpx3cIj4nHbHSlK3Ldc3NhiT4,2667
 verifiers/v1/configs/cli/eval.py,sha256=9Zw7mAUipMhoIcwRa_lgo5gII4rMmMEBAgzcPxPWPjw,6895
-verifiers/v1/configs/cli/init.py,sha256=JvIUOdhfZfNOXg2wm2i06jLLHpb97tNMzZqmcK_9wVw,916
 verifiers/v1/configs/cli/replay.py,sha256=vUCNs-9j1CXcAZhGozN1EaztcXK_dYVK3iN_fRmbSXw,2929
 verifiers/v1/configs/cli/validate.py,sha256=m5ReEfF4-_hOlsvuCtCdeCDywXAfQoKR-NpH6Fzv1u0,3990
 verifiers/v1/dialects/__init__.py,sha256=MNOYM2NdTlSxGK4VtoEjMtD5zya2MpjPDX5jMvZUWEo,736
@@ -143,12 +141,12 @@ verifiers/v1/mcp/server.py,sha256=Jqi8OMT8DnoZegXqM3UNNRzhVnbdrPEwx0DjOJo_K2A,13
 verifiers/v1/mcp/toolset.py,sha256=G2wzNdEMGLXoQUcrwF0TartJkqokNPI3o7YUOAaaa6U,1007
 verifiers/v1/runtimes/__init__.py,sha256=zOG9BYsK19VRNqvggqh3f6g1_UGIq7VjIBcK8kDlmnQ,3515
 verifiers/v1/runtimes/apptainer.py,sha256=Ru9QJna_l0goIzg71G-4RU_D-7wZUNz139S5TEvbzog,8211
-verifiers/v1/runtimes/base.py,sha256=5tRDqGGnYvpy2qDa-V6K5EuwLKjEeI9U4gZV_8TJQZE,19947
+verifiers/v1/runtimes/base.py,sha256=4ma91e_M1KXOAx74dNBVPJoxrqnQaOuvbkSBlHMmujE,20128
 verifiers/v1/runtimes/container.py,sha256=_hNhGOo1Zvorj7THSj2UrZTkgewySUVJe4Kd777-QnI,12563
 verifiers/v1/runtimes/e2b.py,sha256=yBeocAutsv2l5x-C5IMqrlgBz6Cb-t6vWbX4qTjlztE,30306
 verifiers/v1/runtimes/limiters.py,sha256=rOfQJcMdYcLSKq8AL9O09BfGOLqpYmNgrCxXXNcHLmU,2849
 verifiers/v1/runtimes/modal.py,sha256=ZHA4vtNIZwTV-bnVUlambgc8l34j_ZulvtrwvvkE2Kg,17945
-verifiers/v1/runtimes/prime.py,sha256=EzKfjtBIfH8ehjrvOI4khm6BIEZ87r84B5gmgFJQvU4,20836
+verifiers/v1/runtimes/prime.py,sha256=xo9vFn6gNQRhG_8DFPcSyKcTCcZRf4n5UPS7cnbRFm8,20880
 verifiers/v1/runtimes/subprocess.py,sha256=vnGVAI3J7pYmRei4MQ7MN4nb8SglLi70NtVTINMPVvQ,8855
 verifiers/v1/runtimes/docker/__init__.py,sha256=QoFOTMgRjoWD--O25Jsft9eD2vjcrMpJQyhfQXqWraI,21828
 verifiers/v1/runtimes/docker/egress.py,sha256=wtKWTL2_zFidxMR7lDRmfevQlOsl2Iyk88ebdO4Kqqw,24566
@@ -185,7 +183,7 @@ verifiers/v1/utils/generic.py,sha256=isejTTgRoFDJQOM18L3sWqfzBRQyYUkQvk-gNbrYAVc
 verifiers/v1/utils/git.py,sha256=MTRinOvG4EddfjdahHkFZUihHmhJ37n5ROtgjzjE1NA,8366
 verifiers/v1/utils/image.py,sha256=OFw_wdwVtbdwa8uJ3X3vYfhpUAi0CTH5HMBupjVsRRQ,282
 verifiers/v1/utils/interrupt.py,sha256=F-KKhc5ndPJJfhd3SuMqyqhXhA32FhCRy5KWFJpEoM4,1179
-verifiers/v1/utils/loaders.py,sha256=2TrgZK3MzKub1GnfaTVIQhlwF6q4qATXZCjicH6nT9c,12299
+verifiers/v1/utils/loaders.py,sha256=WhIHRnAXfFpfCEgH35J1XHd95KRarmg62nxUkUB4HME,12670
 verifiers/v1/utils/logging.py,sha256=-zgaM-HZsgxumZ4wYNlRMZepp6JanCSjeL2wSVulwJM,2296
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/paths.py,sha256=it_4JWf_8bPZe4TRcyvL7_KJ4fsbEsxhhIY_xA8Ikus,308
@@ -196,8 +194,8 @@ verifiers/v1/utils/scope.py,sha256=bzUEiWDOVdbj7dXOwGOlzsQ_-Xu0j9NR5TalQB-8TS8,8
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/trace_store.py,sha256=RsCDonGR-bs-tvEstcppiJm3jFmAdIiG83fA5KtCbAw,2801
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.4.1.dev3.dist-info/METADATA,sha256=0csv8E7xavXWUlwAD-nPkvlQUl3txcdVBDBDbDpFIBI,4221
-verifiers-0.4.1.dev3.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-verifiers-0.4.1.dev3.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.4.1.dev3.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.4.1.dev3.dist-info/RECORD,,
+verifiers-0.4.1.dev5.dist-info/METADATA,sha256=12xwkEJs-6xoBi01_gAkwto_zrPPL5Ps77wBUJ9UXNA,5361
+verifiers-0.4.1.dev5.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+verifiers-0.4.1.dev5.dist-info/entry_points.txt,sha256=O1bIO0gyJWuLfWtTGF0h-JkNz6FkyXP7Z9iMFYnjeMI,222
+verifiers-0.4.1.dev5.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.4.1.dev5.dist-info/RECORD,,

{verifiers-0.4.1.dev3.dist-info → verifiers-0.4.1.dev5.dist-info}/entry_points.txt

@@ -2,6 +2,5 @@
 vf-debug = verifiers.v1.cli.debug:main
 vf-eval = verifiers.v1.cli.eval.main:main
 vf-gepa = verifiers.v1.cli.gepa:main
-vf-init = verifiers.v1.cli.init:main
 vf-replay = verifiers.v1.cli.replay:main
 vf-validate = verifiers.v1.cli.validate:main

verifiers/v1/cli/init.py

@@ -1,252 +0,0 @@
-"""Scaffold v1 environment packages."""
-
-import sys
-from pathlib import Path
-
-from pydantic_config import cli
-
-from verifiers.v1.configs.cli.init import InitConfig
-
-USAGE = (
-    "usage: uv run vf-init <name> [--path ./environments] [-T/--add-tool] "
-    "[-H/--add-harness]\n"
-    "       scaffold a new environment package"
-)
-
-
-def _names(name: str) -> tuple[str, str, str, str]:
-    dash = name.strip().strip("/").replace("_", "-").lower()
-    pkg = dash.replace("-", "_")
-    stem = pkg.removesuffix("_v1")
-    prefix = "".join(part[:1].upper() + part[1:] for part in stem.split("_") if part)
-    if not prefix or not prefix[0].isalpha():
-        prefix = f"Env{prefix}"
-    return dash, pkg, stem, prefix
-
-
-def _write(path: Path, content: str) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(content)
-    print(f"  write  {path}")
-
-
-def _pyproject(dash: str, pkg: str) -> str:
-    return f"""\
-[project]
-name = "{dash}"
-version = "0.1.0"
-description = "{dash} — <one-line description>."
-requires-python = ">=3.11"
-dependencies = ["verifiers"]
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.wheel]
-packages = ["{pkg}"]
-"""
-
-
-def _init_py(pkg: str, prefix: str, add_harness: bool) -> str:
-    lines = [f"from {pkg}.taskset import {prefix}Taskset"]
-    exports = [f"{prefix}Taskset"]
-    if add_harness:
-        lines.append(f"from {pkg}.harness import {prefix}Harness")
-        exports.append(f"{prefix}Harness")
-    exports_repr = ", ".join(f'"{e}"' for e in exports)
-    return "\n".join(lines) + f"\n\n__all__ = [{exports_repr}]\n"
-
-
-def _taskset_py(pkg: str, prefix: str, *, add_tool: bool) -> str:
-    imports = "import verifiers.v1 as vf"
-    local_imports: list[str] = []
-    task_config_fields = ""
-    task_decls = ""
-    if add_tool:
-        local_imports.append(f"from {pkg}.servers.tool import {prefix}Toolset")
-        task_config_fields += "\n    tools: vf.ToolsetConfig = vf.ToolsetConfig()"
-        task_decls += (
-            "\n\n    @classmethod"
-            f"\n    def toolsets(cls, config: {prefix}TaskConfig) -> list[vf.Toolset]:"
-            f"\n        return [{prefix}Toolset(config.tools)]"
-            "\n"
-        )
-    if local_imports:
-        imports += "\n\n" + "\n".join(local_imports)
-    task_config = (
-        f"\n\nclass {prefix}TaskConfig(vf.TaskConfig):\n"
-        '    """Knobs the task reads from ``self.config``; configure them under '
-        f'``--env.taskset.task.*``."""{task_config_fields}\n'
-        if add_tool
-        else ""
-    )
-    task_generic = (
-        f"{prefix}Data, vf.State, {prefix}TaskConfig" if add_tool else f"{prefix}Data"
-    )
-    config_body = '    num_tasks: int = 5\n    """How many tasks to build."""\n' + (
-        f"    task: {prefix}TaskConfig = {prefix}TaskConfig()\n" if add_tool else ""
-    )
-    return f'''\
-{imports}
-
-
-class {prefix}Data(vf.TaskData):
-    """One row's data. Add task-specific fields here, such as a reference answer."""
-{task_config}
-
-class {prefix}Task(vf.Task[{task_generic}]):
-    """Rewards, hooks, and servers, with row data available on ``self.data``."""{task_decls}
-    @vf.reward(weight=1.0)
-    async def reward(self, trace: vf.Trace) -> float:
-        raise NotImplementedError("Score the rollout and return a float (e.g. in [0, 1]).")
-
-
-class {prefix}Config(vf.TasksetConfig):
-{config_body}
-
-class {prefix}Taskset(vf.Taskset[{prefix}Task, {prefix}Config]):
-    def load(self) -> list[{prefix}Task]:
-        raise NotImplementedError(
-            "Return this taskset's tasks, e.g. "
-            "[{prefix}Task({prefix}Data(prompt=...), self.config.task) "
-            "for i in range(self.config.num_tasks)]."
-        )
-'''
-
-
-def _tool_py(stem: str, prefix: str) -> str:
-    return f'''\
-import verifiers.v1 as vf
-
-
-class {prefix}Toolset(vf.Toolset[vf.ToolsetConfig]):
-    TOOL_PREFIX = "{stem}"
-
-    @vf.tool
-    def echo(self, text: str) -> str:
-        """Return the given text unchanged (replace with a real tool)."""
-        return text
-
-
-if __name__ == "__main__":
-    {prefix}Toolset.run()
-'''
-
-
-def _harness_py(prefix: str) -> str:
-    return f'''\
-import verifiers.v1 as vf
-
-
-class {prefix}HarnessConfig(vf.HarnessConfig):
-    """Run knobs for this harness. Add fields here (e.g. a CLI version to install)."""
-
-
-class {prefix}Harness(vf.Harness[{prefix}HarnessConfig]):
-    APPENDS_SYSTEM_PROMPT = True
-
-    async def launch(
-        self,
-        ctx: vf.ModelContext,
-        trace: vf.Trace,
-        runtime: vf.Runtime,
-        endpoint: str,
-        secret: str,
-        mcp_urls: dict[str, str],
-        data: vf.TaskData,
-    ) -> vf.ProgramResult:
-        raise NotImplementedError(
-            "Implement launch: run your agent in `runtime`, pointing its model calls at "
-            "`endpoint` with bearer token `secret`."
-        )
-'''
-
-
-def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str:
-    layout = [
-        (
-            f"- `{pkg}/taskset.py` — the task (`@reward` scoring + behavior) and the taskset: "
-            "`load` (data + prompts)."
-        )
-    ]
-    if add_tool:
-        layout.append(
-            f"- `{pkg}/servers/tool.py` — a `vf.Toolset` tool server, constructed in `Task.toolsets`."
-        )
-    if add_harness:
-        layout.append(
-            f"- `{pkg}/harness.py` — a custom harness, selectable with `--env.agent.harness.id {dash}`."
-        )
-    layout_block = "\n".join(layout)
-    return f"""\
-# {dash}
-
-A v1 verifiers environment, scaffolded with `init`.
-
-## Develop
-
-1. Implement `load` and the `@reward` in `{pkg}/taskset.py` (see `environments/`).
-2. Install + run:
-
-```bash
-uv pip install -e .        # install this package (or register it in your project)
-uv run vf-eval {dash} -n 3    # evaluate a few tasks with the bash harness
-```
-
-## Layout
-
-{layout_block}
-
-Tune knobs from the CLI: `--env.taskset.num-tasks 10`, `--model <id>`, `-n`, and `-r`.
-"""
-
-
-def scaffold(config: InitConfig) -> Path:
-    dash, pkg, stem, prefix = _names(config.name)
-    env_dir = Path(config.path) / pkg
-    pkg_dir = env_dir / pkg
-    if env_dir.exists() and not config.force:
-        raise SystemExit(
-            f"error: {env_dir} already exists - refusing to overwrite (pass --force to overwrite)"
-        )
-    print(f"scaffolding v1 environment {dash!r} in {env_dir}")
-
-    _write(env_dir / "pyproject.toml", _pyproject(dash, pkg))
-    _write(
-        env_dir / "README.md",
-        _readme(dash, pkg, add_tool=config.add_tool, add_harness=config.add_harness),
-    )
-    _write(pkg_dir / "__init__.py", _init_py(pkg, prefix, config.add_harness))
-    _write(
-        pkg_dir / "taskset.py",
-        _taskset_py(pkg, prefix, add_tool=config.add_tool),
-    )
-    if config.add_harness:
-        _write(pkg_dir / "harness.py", _harness_py(prefix))
-    if config.add_tool:
-        _write(pkg_dir / "servers" / "__init__.py", "")
-        _write(pkg_dir / "servers" / "tool.py", _tool_py(stem, prefix))
-
-    print(f"\ndone. next:\n  uv pip install -e {env_dir}\n  uv run vf-eval {dash} -n 3")
-    return env_dir
-
-
-def main(argv: list[str] | None = None) -> None:
-    argv = list(sys.argv[1:]) if argv is None else list(argv)
-    if argv and not argv[0].startswith(("-", "@")):
-        argv = ["--name", argv[0], *argv[1:]]
-
-    if not argv or any(arg in ("-h", "--help") for arg in argv):
-        print(USAGE)
-        sys.argv = [sys.argv[0], "--help"]
-        cli(InitConfig)
-        return
-
-    sys.argv = [sys.argv[0], *argv]
-    config = cli(InitConfig)
-    scaffold(config)
-
-
-if __name__ == "__main__":
-    main()

verifiers/v1/configs/cli/init.py

@@ -1,17 +0,0 @@
-"""Environment-scaffold CLI configuration."""
-
-from pydantic import AliasChoices, Field
-from pydantic_config import BaseConfig
-
-
-class InitConfig(BaseConfig):
-    name: str = Field(min_length=1)
-    """The new environment id, e.g. `my-task-v1` (positional: `init my-task-v1`)."""
-    path: str = Field("./environments", validation_alias=AliasChoices("path", "p"))
-    """Parent directory the package is created in (default `./environments`)."""
-    add_tool: bool = Field(False, validation_alias=AliasChoices("add_tool", "T"))
-    """Also scaffold a `vf.Toolset` declared on the task (`-T`)."""
-    add_harness: bool = Field(False, validation_alias=AliasChoices("add_harness", "H"))
-    """Also scaffold a custom `vf.Harness` (`harness.py`), selectable via `--env.agent.harness.id <name>` (`-H`)."""
-    force: bool = False
-    """Overwrite an existing environment package (default: refuse if it already exists)."""