verifiers 0.4.1.dev4__py3-none-any.whl → 0.4.1.dev5__py3-none-any.whl

{verifiers-0.4.1.dev4.dist-info → verifiers-0.4.1.dev5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.4.1.dev4
+Version: 0.4.1.dev5
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.4.1.dev4.dist-info → verifiers-0.4.1.dev5.dist-info}/RECORD

@@ -20,7 +20,6 @@ verifiers/v1/acp/runner.py,sha256=sFsg0UgWeDjpjihejUidIV2yTapVj1Ap0cDTaEktjkI,15
 verifiers/v1/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/cli/debug.py,sha256=OgzXCW9HaTNruZil4f70WP5tR6toKliKaVvV98C5qzE,11665
 verifiers/v1/cli/gepa.py,sha256=IyS3YZCCHVydb-OyPeGdtCYsz6OYqXXddXqYq8RLucM,4374
-verifiers/v1/cli/init.py,sha256=k1wawdQBquiiepPrb0enBT-vdKvfSievgs4CiwSdFUc,7601
 verifiers/v1/cli/output.py,sha256=4FRuQ6UIZCigMfUUO2nCFFJKxls3DtWDXswMCVTxjIA,5010
 verifiers/v1/cli/replay.py,sha256=LkeaV8Sxa7kHue9JKHDg0V_N6KXOcjN_r6RmeQVcxfk,10396
 verifiers/v1/cli/resolve.py,sha256=Q1EHM7wWQo0YkwJA898qtZbYLaIkjIVq3Q7kUp2YIxs,4346
@@ -57,7 +56,6 @@ verifiers/v1/configs/cli/__init__.py,sha256=YMDUPdbRlDJ77dhj3QqzmyRhqNgUvuQdpj_b
 verifiers/v1/configs/cli/debug.py,sha256=CcUZ9MEcv0iAL9w2BkTfnZg20yWJ-Vn8P30-e5tuCic,2741
 verifiers/v1/configs/cli/env.py,sha256=qg2LNOasAU0yg3b0xqXpx3cIj4nHbHSlK3Ldc3NhiT4,2667
 verifiers/v1/configs/cli/eval.py,sha256=9Zw7mAUipMhoIcwRa_lgo5gII4rMmMEBAgzcPxPWPjw,6895
-verifiers/v1/configs/cli/init.py,sha256=JvIUOdhfZfNOXg2wm2i06jLLHpb97tNMzZqmcK_9wVw,916
 verifiers/v1/configs/cli/replay.py,sha256=vUCNs-9j1CXcAZhGozN1EaztcXK_dYVK3iN_fRmbSXw,2929
 verifiers/v1/configs/cli/validate.py,sha256=m5ReEfF4-_hOlsvuCtCdeCDywXAfQoKR-NpH6Fzv1u0,3990
 verifiers/v1/dialects/__init__.py,sha256=MNOYM2NdTlSxGK4VtoEjMtD5zya2MpjPDX5jMvZUWEo,736
@@ -196,8 +194,8 @@ verifiers/v1/utils/scope.py,sha256=bzUEiWDOVdbj7dXOwGOlzsQ_-Xu0j9NR5TalQB-8TS8,8
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/trace_store.py,sha256=RsCDonGR-bs-tvEstcppiJm3jFmAdIiG83fA5KtCbAw,2801
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.4.1.dev4.dist-info/METADATA,sha256=5NhhAb0-7bgsTf7b1_EgiLAHQ26JMkqNFBmmmR5dsUs,5361
-verifiers-0.4.1.dev4.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-verifiers-0.4.1.dev4.dist-info/entry_points.txt,sha256=iugElcdWPKbQM7uFF0lZ8iUpHsNr17-BwEAAjJWxV3U,259
-verifiers-0.4.1.dev4.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.4.1.dev4.dist-info/RECORD,,
+verifiers-0.4.1.dev5.dist-info/METADATA,sha256=12xwkEJs-6xoBi01_gAkwto_zrPPL5Ps77wBUJ9UXNA,5361
+verifiers-0.4.1.dev5.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+verifiers-0.4.1.dev5.dist-info/entry_points.txt,sha256=O1bIO0gyJWuLfWtTGF0h-JkNz6FkyXP7Z9iMFYnjeMI,222
+verifiers-0.4.1.dev5.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.4.1.dev5.dist-info/RECORD,,

{verifiers-0.4.1.dev4.dist-info → verifiers-0.4.1.dev5.dist-info}/entry_points.txt

@@ -2,6 +2,5 @@
 vf-debug = verifiers.v1.cli.debug:main
 vf-eval = verifiers.v1.cli.eval.main:main
 vf-gepa = verifiers.v1.cli.gepa:main
-vf-init = verifiers.v1.cli.init:main
 vf-replay = verifiers.v1.cli.replay:main
 vf-validate = verifiers.v1.cli.validate:main

verifiers/v1/cli/init.py

@@ -1,252 +0,0 @@
-"""Scaffold v1 environment packages."""
-
-import sys
-from pathlib import Path
-
-from pydantic_config import cli
-
-from verifiers.v1.configs.cli.init import InitConfig
-
-USAGE = (
-    "usage: uv run vf-init <name> [--path ./environments] [-T/--add-tool] "
-    "[-H/--add-harness]\n"
-    "       scaffold a new environment package"
-)
-
-
-def _names(name: str) -> tuple[str, str, str, str]:
-    dash = name.strip().strip("/").replace("_", "-").lower()
-    pkg = dash.replace("-", "_")
-    stem = pkg.removesuffix("_v1")
-    prefix = "".join(part[:1].upper() + part[1:] for part in stem.split("_") if part)
-    if not prefix or not prefix[0].isalpha():
-        prefix = f"Env{prefix}"
-    return dash, pkg, stem, prefix
-
-
-def _write(path: Path, content: str) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(content)
-    print(f"  write  {path}")
-
-
-def _pyproject(dash: str, pkg: str) -> str:
-    return f"""\
-[project]
-name = "{dash}"
-version = "0.1.0"
-description = "{dash} — <one-line description>."
-requires-python = ">=3.11"
-dependencies = ["verifiers"]
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
-
-[tool.hatch.build.targets.wheel]
-packages = ["{pkg}"]
-"""
-
-
-def _init_py(pkg: str, prefix: str, add_harness: bool) -> str:
-    lines = [f"from {pkg}.taskset import {prefix}Taskset"]
-    exports = [f"{prefix}Taskset"]
-    if add_harness:
-        lines.append(f"from {pkg}.harness import {prefix}Harness")
-        exports.append(f"{prefix}Harness")
-    exports_repr = ", ".join(f'"{e}"' for e in exports)
-    return "\n".join(lines) + f"\n\n__all__ = [{exports_repr}]\n"
-
-
-def _taskset_py(pkg: str, prefix: str, *, add_tool: bool) -> str:
-    imports = "import verifiers.v1 as vf"
-    local_imports: list[str] = []
-    task_config_fields = ""
-    task_decls = ""
-    if add_tool:
-        local_imports.append(f"from {pkg}.servers.tool import {prefix}Toolset")
-        task_config_fields += "\n    tools: vf.ToolsetConfig = vf.ToolsetConfig()"
-        task_decls += (
-            "\n\n    @classmethod"
-            f"\n    def toolsets(cls, config: {prefix}TaskConfig) -> list[vf.Toolset]:"
-            f"\n        return [{prefix}Toolset(config.tools)]"
-            "\n"
-        )
-    if local_imports:
-        imports += "\n\n" + "\n".join(local_imports)
-    task_config = (
-        f"\n\nclass {prefix}TaskConfig(vf.TaskConfig):\n"
-        '    """Knobs the task reads from ``self.config``; configure them under '
-        f'``--env.taskset.task.*``."""{task_config_fields}\n'
-        if add_tool
-        else ""
-    )
-    task_generic = (
-        f"{prefix}Data, vf.State, {prefix}TaskConfig" if add_tool else f"{prefix}Data"
-    )
-    config_body = '    num_tasks: int = 5\n    """How many tasks to build."""\n' + (
-        f"    task: {prefix}TaskConfig = {prefix}TaskConfig()\n" if add_tool else ""
-    )
-    return f'''\
-{imports}
-
-
-class {prefix}Data(vf.TaskData):
-    """One row's data. Add task-specific fields here, such as a reference answer."""
-{task_config}
-
-class {prefix}Task(vf.Task[{task_generic}]):
-    """Rewards, hooks, and servers, with row data available on ``self.data``."""{task_decls}
-    @vf.reward(weight=1.0)
-    async def reward(self, trace: vf.Trace) -> float:
-        raise NotImplementedError("Score the rollout and return a float (e.g. in [0, 1]).")
-
-
-class {prefix}Config(vf.TasksetConfig):
-{config_body}
-
-class {prefix}Taskset(vf.Taskset[{prefix}Task, {prefix}Config]):
-    def load(self) -> list[{prefix}Task]:
-        raise NotImplementedError(
-            "Return this taskset's tasks, e.g. "
-            "[{prefix}Task({prefix}Data(prompt=...), self.config.task) "
-            "for i in range(self.config.num_tasks)]."
-        )
-'''
-
-
-def _tool_py(stem: str, prefix: str) -> str:
-    return f'''\
-import verifiers.v1 as vf
-
-
-class {prefix}Toolset(vf.Toolset[vf.ToolsetConfig]):
-    TOOL_PREFIX = "{stem}"
-
-    @vf.tool
-    def echo(self, text: str) -> str:
-        """Return the given text unchanged (replace with a real tool)."""
-        return text
-
-
-if __name__ == "__main__":
-    {prefix}Toolset.run()
-'''
-
-
-def _harness_py(prefix: str) -> str:
-    return f'''\
-import verifiers.v1 as vf
-
-
-class {prefix}HarnessConfig(vf.HarnessConfig):
-    """Run knobs for this harness. Add fields here (e.g. a CLI version to install)."""
-
-
-class {prefix}Harness(vf.Harness[{prefix}HarnessConfig]):
-    APPENDS_SYSTEM_PROMPT = True
-
-    async def launch(
-        self,
-        ctx: vf.ModelContext,
-        trace: vf.Trace,
-        runtime: vf.Runtime,
-        endpoint: str,
-        secret: str,
-        mcp_urls: dict[str, str],
-        data: vf.TaskData,
-    ) -> vf.ProgramResult:
-        raise NotImplementedError(
-            "Implement launch: run your agent in `runtime`, pointing its model calls at "
-            "`endpoint` with bearer token `secret`."
-        )
-'''
-
-
-def _readme(dash: str, pkg: str, *, add_tool: bool, add_harness: bool) -> str:
-    layout = [
-        (
-            f"- `{pkg}/taskset.py` — the task (`@reward` scoring + behavior) and the taskset: "
-            "`load` (data + prompts)."
-        )
-    ]
-    if add_tool:
-        layout.append(
-            f"- `{pkg}/servers/tool.py` — a `vf.Toolset` tool server, constructed in `Task.toolsets`."
-        )
-    if add_harness:
-        layout.append(
-            f"- `{pkg}/harness.py` — a custom harness, selectable with `--env.agent.harness.id {dash}`."
-        )
-    layout_block = "\n".join(layout)
-    return f"""\
-# {dash}
-
-A v1 verifiers environment, scaffolded with `init`.
-
-## Develop
-
-1. Implement `load` and the `@reward` in `{pkg}/taskset.py` (see `environments/`).
-2. Install + run:
-
-```bash
-uv pip install -e .        # install this package (or register it in your project)
-uv run vf-eval {dash} -n 3    # evaluate a few tasks with the bash harness
-```
-
-## Layout
-
-{layout_block}
-
-Tune knobs from the CLI: `--env.taskset.num-tasks 10`, `--model <id>`, `-n`, and `-r`.
-"""
-
-
-def scaffold(config: InitConfig) -> Path:
-    dash, pkg, stem, prefix = _names(config.name)
-    env_dir = Path(config.path) / pkg
-    pkg_dir = env_dir / pkg
-    if env_dir.exists() and not config.force:
-        raise SystemExit(
-            f"error: {env_dir} already exists - refusing to overwrite (pass --force to overwrite)"
-        )
-    print(f"scaffolding v1 environment {dash!r} in {env_dir}")
-
-    _write(env_dir / "pyproject.toml", _pyproject(dash, pkg))
-    _write(
-        env_dir / "README.md",
-        _readme(dash, pkg, add_tool=config.add_tool, add_harness=config.add_harness),
-    )
-    _write(pkg_dir / "__init__.py", _init_py(pkg, prefix, config.add_harness))
-    _write(
-        pkg_dir / "taskset.py",
-        _taskset_py(pkg, prefix, add_tool=config.add_tool),
-    )
-    if config.add_harness:
-        _write(pkg_dir / "harness.py", _harness_py(prefix))
-    if config.add_tool:
-        _write(pkg_dir / "servers" / "__init__.py", "")
-        _write(pkg_dir / "servers" / "tool.py", _tool_py(stem, prefix))
-
-    print(f"\ndone. next:\n  uv pip install -e {env_dir}\n  uv run vf-eval {dash} -n 3")
-    return env_dir
-
-
-def main(argv: list[str] | None = None) -> None:
-    argv = list(sys.argv[1:]) if argv is None else list(argv)
-    if argv and not argv[0].startswith(("-", "@")):
-        argv = ["--name", argv[0], *argv[1:]]
-
-    if not argv or any(arg in ("-h", "--help") for arg in argv):
-        print(USAGE)
-        sys.argv = [sys.argv[0], "--help"]
-        cli(InitConfig)
-        return
-
-    sys.argv = [sys.argv[0], *argv]
-    config = cli(InitConfig)
-    scaffold(config)
-
-
-if __name__ == "__main__":
-    main()

verifiers/v1/configs/cli/init.py

@@ -1,17 +0,0 @@
-"""Environment-scaffold CLI configuration."""
-
-from pydantic import AliasChoices, Field
-from pydantic_config import BaseConfig
-
-
-class InitConfig(BaseConfig):
-    name: str = Field(min_length=1)
-    """The new environment id, e.g. `my-task-v1` (positional: `init my-task-v1`)."""
-    path: str = Field("./environments", validation_alias=AliasChoices("path", "p"))
-    """Parent directory the package is created in (default `./environments`)."""
-    add_tool: bool = Field(False, validation_alias=AliasChoices("add_tool", "T"))
-    """Also scaffold a `vf.Toolset` declared on the task (`-T`)."""
-    add_harness: bool = Field(False, validation_alias=AliasChoices("add_harness", "H"))
-    """Also scaffold a custom `vf.Harness` (`harness.py`), selectable via `--env.agent.harness.id <name>` (`-H`)."""
-    force: bool = False
-    """Overwrite an existing environment package (default: refuse if it already exists)."""