worldevals 0.3.0__py3-none-any.whl

worldevals/__init__.py

@@ -0,0 +1,30 @@
+"""WorldEvals — the Inspect Evals for robotics.
+
+A curated catalog of physical-AI benchmarks built on
+`Inspect Robots <https://github.com/robocurve/inspect-robots>`_. Each benchmark is its own
+repo (e.g. `KitchenBench <https://github.com/robocurve/kitchenbench>`_); this
+package indexes them and bridges "what exists" with "what's installed".
+"""
+
+from __future__ import annotations
+
+from worldevals.catalog import (
+    CATALOG,
+    Benchmark,
+    benchmark_for_task,
+    by_tag,
+    catalog,
+    get,
+)
+
+__version__ = "0.3.0"
+
+__all__ = [
+    "CATALOG",
+    "Benchmark",
+    "__version__",
+    "benchmark_for_task",
+    "by_tag",
+    "catalog",
+    "get",
+]

worldevals/catalog.py

@@ -0,0 +1,90 @@
+"""The WorldEvals catalog — the registry of physical-AI benchmark repos.
+
+Each benchmark is its own repository (built on Inspect Robots) that registers its tasks
+via entry points. WorldEvals indexes them so you can discover what exists and how
+to install it. To add a benchmark, append a
+[`Benchmark`][worldevals.catalog.Benchmark] entry here (PR).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+
+@dataclass(frozen=True)
+class Benchmark:
+    """One benchmark repo in the WorldEvals collection."""
+
+    name: str
+    title: str
+    description: str
+    repo: str
+    install: str
+    task_keys: tuple[str, ...]
+    tags: tuple[str, ...]
+    bimanual: bool
+    contributors: tuple[str, ...]
+    status: Literal["alpha", "beta", "stable"] = "alpha"
+
+
+_KITCHENBENCH_TASKS = (
+    "kitchenbench/place_cutlery",
+    "kitchenbench/stack",
+    "kitchenbench/place_in_rack",
+    "kitchenbench/pour_pasta",
+    "kitchenbench/open_container",
+    "kitchenbench/fold_cloth",
+    "kitchenbench/seal_container",
+    "kitchenbench/handoff",
+    "kitchenbench/sort_cutlery",
+    "kitchenbench/scoop_pasta",
+)
+
+CATALOG: tuple[Benchmark, ...] = (
+    Benchmark(
+        name="kitchenbench",
+        title="KitchenBench",
+        description=(
+            "10 bimanual kitchen-manipulation tasks: pick-place, stacking, slotted "
+            "insertion, granular pour & tool-scoop, lid open/seal, cloth folding, a "
+            "two-arm handover, and a multi-instance cutlery sort."
+        ),
+        repo="https://github.com/robocurve/kitchenbench",
+        install=(
+            'pip install "inspect-robots @ git+https://github.com/robocurve/inspect-robots@v0.3.0"'
+            ' && pip install "kitchenbench @ git+https://github.com/robocurve/kitchenbench"'
+        ),
+        task_keys=_KITCHENBENCH_TASKS,
+        tags=("kitchen", "bimanual", "manipulation"),
+        bimanual=True,
+        contributors=("robocurve",),
+        status="alpha",
+    ),
+)
+
+
+def catalog() -> tuple[Benchmark, ...]:
+    """All benchmarks in the collection."""
+    return CATALOG
+
+
+def get(name: str) -> Benchmark:
+    """Look up a benchmark by name; raise `KeyError` if unknown."""
+    for benchmark in CATALOG:
+        if benchmark.name == name:
+            return benchmark
+    raise KeyError(f"no benchmark named {name!r}; known: {sorted(b.name for b in CATALOG)}")
+
+
+def by_tag(tag: str) -> list[Benchmark]:
+    """All benchmarks carrying ``tag``."""
+    return [b for b in CATALOG if tag in b.tags]
+
+
+def benchmark_for_task(task_key: str) -> Benchmark | None:
+    """The benchmark that registers ``task_key``, or ``None`` if not in the catalog."""
+    for benchmark in CATALOG:
+        if task_key in benchmark.task_keys:
+            return benchmark
+    return None

worldevals/cli.py

@@ -0,0 +1,83 @@
+"""The ``worldevals`` CLI: browse the benchmark catalog and see what's installed."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Sequence
+
+from worldevals.catalog import benchmark_for_task, by_tag, catalog, get
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="worldevals",
+        description="WorldEvals — the Inspect Evals for robotics (a benchmark catalog).",
+    )
+    sub = parser.add_subparsers(dest="command")
+
+    p_list = sub.add_parser("list", help="list catalogued benchmarks")
+    p_list.add_argument("--tag", help="only benchmarks carrying this tag")
+
+    p_info = sub.add_parser("info", help="show details for one benchmark")
+    p_info.add_argument("name")
+
+    sub.add_parser("tasks", help="Inspect Robots tasks installed locally, by benchmark")
+    return parser
+
+
+def _cmd_list(tag: str | None) -> int:
+    benchmarks = by_tag(tag) if tag else list(catalog())
+    if not benchmarks:
+        print("(no benchmarks)" if not tag else f"(no benchmarks tagged {tag!r})")
+        return 0
+    for b in benchmarks:
+        print(f"{b.name}  [{b.status}]  {b.title}")
+        print(f"  {len(b.task_keys)} tasks · tags: {', '.join(b.tags)}")
+    return 0
+
+
+def _cmd_info(name: str) -> int:
+    try:
+        b = get(name)
+    except KeyError as exc:
+        print(str(exc))
+        return 1
+    print(f"{b.title} ({b.name})  [{b.status}]")
+    print(f"  {b.description}")
+    print(f"  repo:    {b.repo}")
+    print(f"  install: {b.install}")
+    print(f"  bimanual: {b.bimanual}   tags: {', '.join(b.tags)}")
+    print("  tasks:")
+    for key in b.task_keys:
+        print(f"    - {key}")
+    return 0
+
+
+def _cmd_tasks() -> int:
+    from inspect_robots.registry import registered
+
+    tasks = sorted(registered("task"))
+    if not tasks:
+        print("(no Inspect Robots tasks installed)")
+        return 0
+    for key in tasks:
+        b = benchmark_for_task(key)
+        print(f"{key}  ←  {b.name if b is not None else '—'}")
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    if args.command == "list":
+        return _cmd_list(args.tag)
+    if args.command == "info":
+        return _cmd_info(args.name)
+    if args.command == "tasks":
+        return _cmd_tasks()
+    parser.print_help()
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

worldevals-0.3.0.dist-info/METADATA

@@ -0,0 +1,148 @@
+Metadata-Version: 2.4
+Name: worldevals
+Version: 0.3.0
+Summary: The Inspect Evals for robotics — a curated catalog of physical-AI benchmarks built on Inspect Robots.
+Project-URL: Homepage, https://github.com/robocurve/worldevals
+Project-URL: Documentation, https://worldevals.org/
+Project-URL: Framework, https://github.com/robocurve/inspect-robots
+Author: RoboCurve
+License-Expression: MIT
+License-File: LICENSE
+Keywords: benchmarks,catalog,evaluation,physical-ai,robotics,vla
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Typing :: Typed
+Requires-Python: >=3.10
+Requires-Dist: inspect-robots>=0.3
+Provides-Extra: dev
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pre-commit>=3.5; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-gen-files<0.6.1,>=0.5; extra == 'docs'
+Requires-Dist: mkdocs-llmstxt>=0.2; extra == 'docs'
+Requires-Dist: mkdocs-material>=9.5; extra == 'docs'
+Requires-Dist: mkdocs>=1.6; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.26; extra == 'docs'
+Description-Content-Type: text/markdown
+
+<div align="center">
+
+# 🌍 WorldEvals
+
+**The [Inspect Evals](https://inspect.aisi.org.uk/evals/) for robotics.**
+
+A curated catalog of physical-AI / VLA benchmarks built on
+[Inspect Robots](https://github.com/robocurve/inspect-robots).
+
+[![CI](https://github.com/robocurve/worldevals/actions/workflows/ci.yml/badge.svg)](https://github.com/robocurve/worldevals/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
+[![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen)](https://github.com/robocurve/worldevals/actions/workflows/ci.yml)
+[![Built on Inspect Robots](https://img.shields.io/badge/built%20on-Inspect%20Robots-indigo)](https://github.com/robocurve/inspect-robots)
+
+**[📖 Browse the catalog → worldevals.org](https://worldevals.org/)**
+
+</div>
+
+[Inspect Robots](https://github.com/robocurve/inspect-robots) is the *framework* (the "Inspect
+AI for robotics"). **WorldEvals is the collection** — but unlike Inspect Evals'
+monorepo, each benchmark here lives in **its own repository** so it owns its
+release cadence, dependencies, hardware notes, and leaderboard. WorldEvals is the
+lightweight index that ties them together: what benchmarks exist, what tasks each
+provides, and how to install them.
+
+- `inspect-robots list` tells you what's **installed**.
+- `worldevals list` tells you what **exists** and how to get it.
+
+## Benchmarks
+
+| Benchmark | Tasks | Tags | Status |
+|---|--:|---|---|
+| [KitchenBench](https://github.com/robocurve/kitchenbench) — 10 bimanual kitchen-manipulation tasks | 10 | kitchen, bimanual, manipulation | alpha |
+
+## Install & use
+
+```bash
+# Inspect Robots isn't on PyPI yet, so install it from its git tag first:
+pip install "inspect-robots @ git+https://github.com/robocurve/inspect-robots@v0.3.0"
+pip install "worldevals @ git+https://github.com/robocurve/worldevals"
+
+worldevals list                 # all benchmarks
+worldevals list --tag bimanual  # filter by tag
+worldevals info kitchenbench    # repo, install command, task keys
+worldevals tasks                # Inspect Robots tasks installed locally, by benchmark
+```
+
+Then install a benchmark and run it through Inspect Robots:
+
+```bash
+pip install "inspect-robots @ git+https://github.com/robocurve/inspect-robots@v0.3.0"
+pip install "kitchenbench @ git+https://github.com/robocurve/kitchenbench"
+inspect-robots run --task kitchenbench/pour_pasta --policy kitchen_scripted --embodiment kitchen
+```
+
+## Backends (run on real robots)
+
+Benchmarks are embodiment-agnostic; **backend adapters** supply a concrete
+`Policy` + `Embodiment` so a benchmark runs on real hardware or a simulator.
+These are their own repos too (not catalog entries):
+
+| Adapter | Policy · Embodiment | Stack |
+|---|---|---|
+| [inspect-robots-yam](https://github.com/robocurve/inspect-robots-yam) | `molmoact2` · `yam_arms` | [MolmoAct2](https://github.com/allenai/molmoact2) on [I2RT YAM](https://i2rt.com/products/yam-6-dof-arm) bimanual arms |
+| [inspect-robots-so101](https://github.com/robocurve/inspect-robots-so101) | `lerobot` · `so_arm` | [LeRobot](https://github.com/huggingface/lerobot) policies (ACT, SmolVLA, π0, …) on [SO-ARM](https://github.com/TheRobotStudio/SO-ARM100) (SO-100 / SO-101) follower arms |
+
+```bash
+inspect-robots run --task kitchenbench/pour_pasta --policy molmoact2 --embodiment yam_arms
+```
+
+## Add your benchmark
+
+A benchmark is any repo that:
+
+1. depends on `inspect-robots`,
+2. defines one or more Inspect Robots `Task`s, and
+3. registers them via `[project.entry-points."inspect_robots.tasks"]` (and, if it ships
+   a sim/embodiment or policy, `inspect_robots.embodiments` / `inspect_robots.policies`).
+
+To list it here, add a `Benchmark(...)` entry to
+[`src/worldevals/catalog.py`](src/worldevals/catalog.py) and open a PR. A test
+validates every entry (unique name, well-formed repo URL, ≥1 task key). See
+[KitchenBench](https://github.com/robocurve/kitchenbench) as the reference
+implementation.
+
+## Development
+
+```bash
+uv venv && uv pip install -e ".[dev]"     # inspect_robots resolved from the v0.3.0 tag
+uv run pre-commit install
+uv run pytest --cov                        # 100% coverage required
+uv run ruff check . && uv run mypy
+```
+
+## Citation
+
+If you use WorldEvals in your research, please cite it:
+
+```bibtex
+@software{worldevals,
+  author  = {Robocurve},
+  title   = {WorldEvals: A curated catalog of physical-AI benchmarks},
+  year    = {2026},
+  url     = {https://github.com/robocurve/worldevals},
+  version = {0.3.0},
+  license = {MIT}
+}
+```
+
+## License
+
+[MIT](LICENSE)

worldevals-0.3.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+worldevals/__init__.py,sha256=sEvwQeYg2A3D4-ZEaW3E3lsxbwnlmmp9G-raipazfTI,662
+worldevals/catalog.py,sha256=fQeSsC-oxPL1wy9i-OM00Hs9d5q4mSW9n32zhf91Oj0,2796
+worldevals/cli.py,sha256=p_hoaDm7mbxVKc3dPZnMDHkBRw588FbIMYwMVB9x0fU,2472
+worldevals/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+worldevals-0.3.0.dist-info/METADATA,sha256=GbsFGVC45TY2sHI_OgNGTydVtZ_VCmJrjo9AsEmR2pg,6133
+worldevals-0.3.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
+worldevals-0.3.0.dist-info/entry_points.txt,sha256=a0l_2SUgpvHlRMgx8BPa3WmnzvnXDfzSFLyn69dHWuA,51
+worldevals-0.3.0.dist-info/licenses/LICENSE,sha256=NaimrZjfDyMR93d5rbHzWsYQ4UrnVsxgIXZui2UHFjU,1066
+worldevals-0.3.0.dist-info/RECORD,,

worldevals-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.30.1
+Root-Is-Purelib: true
+Tag: py3-none-any

worldevals-0.3.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+worldevals = worldevals.cli:main

worldevals-0.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 robocurve
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.