rua 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rua-0.0.1/PKG-INFO +75 -0
- rua-0.0.1/README.md +62 -0
- rua-0.0.1/pyproject.toml +25 -0
- rua-0.0.1/pyproject.toml.orig +19 -0
- rua-0.0.1/src/rua/__init__.py +119 -0
- rua-0.0.1/src/rua/py.typed +0 -0
rua-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: rua
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: The eval interface for rua-bench: how an agent and a benchmark task talk to each other, and the loop that runs one episode.
|
|
5
|
+
Keywords: rua-bench,benchmark,evaluation,robotics,agents
|
|
6
|
+
Classifier: Development Status :: 2 - Pre-Alpha
|
|
7
|
+
Classifier: Intended Audience :: Science/Research
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
10
|
+
Classifier: Typing :: Typed
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# rua
|
|
15
|
+
|
|
16
|
+
The eval interface for [rua-bench](https://pypi.org/project/rua-bench/), a benchmark for robot-use agents (RUA): LLM agents that control a robot through tool calls.
|
|
17
|
+
|
|
18
|
+
`rua` is a helper for rua-bench. It defines how the agent under evaluation and a benchmark task talk to each other: the task's robot exposes tools and returns observations, the agent picks the next tool call, and `run` plays one episode. rua-bench then scores the episodes across a suite.
|
|
19
|
+
|
|
20
|
+
This is an early release. It defines the interface and the episode loop; the rua-bench tasks and model-backed agents will follow.
|
|
21
|
+
|
|
22
|
+
## Install
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
pip install rua
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## Use
|
|
29
|
+
|
|
30
|
+
A task implements `Robot`:
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from rua import Action, Observation, Tool
|
|
34
|
+
|
|
35
|
+
class Slider:
|
|
36
|
+
"""A one-axis robot that must reach x = 3."""
|
|
37
|
+
|
|
38
|
+
def tools(self):
|
|
39
|
+
return [Tool("move_by", "Move along x", {"type": "object", "properties": {"dx": {"type": "number"}}})]
|
|
40
|
+
|
|
41
|
+
def reset(self):
|
|
42
|
+
self.x = 0.0
|
|
43
|
+
return Observation(text="x=0.0")
|
|
44
|
+
|
|
45
|
+
def step(self, action: Action):
|
|
46
|
+
self.x += action.args["dx"]
|
|
47
|
+
reached = self.x == 3.0
|
|
48
|
+
return Observation(text=f"x={self.x}", done=reached, success=reached or None)
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
Run an agent against it. `ScriptedAgent` replays fixed actions; any object with an `act(observation, tools)` method works, including one that calls an LLM:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from rua import ScriptedAgent, run
|
|
55
|
+
|
|
56
|
+
agent = ScriptedAgent([Action("move_by", {"dx": 1.0}), Action("move_by", {"dx": 2.0})])
|
|
57
|
+
trajectory = run(agent, Slider())
|
|
58
|
+
|
|
59
|
+
trajectory.success # True
|
|
60
|
+
len(trajectory.steps) # 2
|
|
61
|
+
trajectory.last.text # "x=3.0"
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
An episode ends when the robot reports `done`, the agent returns `None`, or `max_steps` is reached. `run` raises `ValueError` if the agent calls a tool the robot does not offer.
|
|
65
|
+
|
|
66
|
+
## Scoring with rua-bench
|
|
67
|
+
|
|
68
|
+
Each trajectory becomes one rua-bench `Episode`:
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
from rua_bench import Episode, Suite, score
|
|
72
|
+
|
|
73
|
+
suite = Suite("demo", ("slider",))
|
|
74
|
+
print(score(suite, [Episode("slider", trajectory.success)]))
|
|
75
|
+
```
|
rua-0.0.1/README.md
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# rua
|
|
2
|
+
|
|
3
|
+
The eval interface for [rua-bench](https://pypi.org/project/rua-bench/), a benchmark for robot-use agents (RUA): LLM agents that control a robot through tool calls.
|
|
4
|
+
|
|
5
|
+
`rua` is a helper for rua-bench. It defines how the agent under evaluation and a benchmark task talk to each other: the task's robot exposes tools and returns observations, the agent picks the next tool call, and `run` plays one episode. rua-bench then scores the episodes across a suite.
|
|
6
|
+
|
|
7
|
+
This is an early release. It defines the interface and the episode loop; the rua-bench tasks and model-backed agents will follow.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install rua
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Use
|
|
16
|
+
|
|
17
|
+
A task implements `Robot`:
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from rua import Action, Observation, Tool
|
|
21
|
+
|
|
22
|
+
class Slider:
|
|
23
|
+
"""A one-axis robot that must reach x = 3."""
|
|
24
|
+
|
|
25
|
+
def tools(self):
|
|
26
|
+
return [Tool("move_by", "Move along x", {"type": "object", "properties": {"dx": {"type": "number"}}})]
|
|
27
|
+
|
|
28
|
+
def reset(self):
|
|
29
|
+
self.x = 0.0
|
|
30
|
+
return Observation(text="x=0.0")
|
|
31
|
+
|
|
32
|
+
def step(self, action: Action):
|
|
33
|
+
self.x += action.args["dx"]
|
|
34
|
+
reached = self.x == 3.0
|
|
35
|
+
return Observation(text=f"x={self.x}", done=reached, success=reached or None)
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Run an agent against it. `ScriptedAgent` replays fixed actions; any object with an `act(observation, tools)` method works, including one that calls an LLM:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
from rua import ScriptedAgent, run
|
|
42
|
+
|
|
43
|
+
agent = ScriptedAgent([Action("move_by", {"dx": 1.0}), Action("move_by", {"dx": 2.0})])
|
|
44
|
+
trajectory = run(agent, Slider())
|
|
45
|
+
|
|
46
|
+
trajectory.success # True
|
|
47
|
+
len(trajectory.steps) # 2
|
|
48
|
+
trajectory.last.text # "x=3.0"
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
An episode ends when the robot reports `done`, the agent returns `None`, or `max_steps` is reached. `run` raises `ValueError` if the agent calls a tool the robot does not offer.
|
|
52
|
+
|
|
53
|
+
## Scoring with rua-bench
|
|
54
|
+
|
|
55
|
+
Each trajectory becomes one rua-bench `Episode`:
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from rua_bench import Episode, Suite, score
|
|
59
|
+
|
|
60
|
+
suite = Suite("demo", ("slider",))
|
|
61
|
+
print(score(suite, [Episode("slider", trajectory.success)]))
|
|
62
|
+
```
|
rua-0.0.1/pyproject.toml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "rua"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "The eval interface for rua-bench: how an agent and a benchmark task talk to each other, and the loop that runs one episode."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
dependencies = []
|
|
8
|
+
keywords = [
|
|
9
|
+
"rua-bench",
|
|
10
|
+
"benchmark",
|
|
11
|
+
"evaluation",
|
|
12
|
+
"robotics",
|
|
13
|
+
"agents",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
|
|
23
|
+
[build-system]
|
|
24
|
+
requires = ["uv_build>=0.13.6,<0.14.0"]
|
|
25
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "rua"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "The eval interface for rua-bench: how an agent and a benchmark task talk to each other, and the loop that runs one episode."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
dependencies = []
|
|
8
|
+
keywords = ["rua-bench", "benchmark", "evaluation", "robotics", "agents"]
|
|
9
|
+
classifiers = [
|
|
10
|
+
"Development Status :: 2 - Pre-Alpha",
|
|
11
|
+
"Intended Audience :: Science/Research",
|
|
12
|
+
"Programming Language :: Python :: 3",
|
|
13
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
14
|
+
"Typing :: Typed",
|
|
15
|
+
]
|
|
16
|
+
|
|
17
|
+
[build-system]
|
|
18
|
+
requires = ["uv_build>=0.13.6,<0.14.0"]
|
|
19
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"""The eval interface for rua-bench: how an agent and a benchmark task talk to each other."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from typing import Any, Protocol, runtime_checkable
|
|
8
|
+
|
|
9
|
+
__version__ = "0.0.1"
|
|
10
|
+
__all__ = [
|
|
11
|
+
"Action",
|
|
12
|
+
"Agent",
|
|
13
|
+
"Observation",
|
|
14
|
+
"Robot",
|
|
15
|
+
"ScriptedAgent",
|
|
16
|
+
"Step",
|
|
17
|
+
"Tool",
|
|
18
|
+
"Trajectory",
|
|
19
|
+
"run",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class Tool:
|
|
25
|
+
"""One call a robot accepts, described the way LLM tool APIs expect."""
|
|
26
|
+
|
|
27
|
+
name: str
|
|
28
|
+
description: str = ""
|
|
29
|
+
parameters: Mapping[str, Any] = field(default_factory=dict) # JSON Schema
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class Action:
|
|
34
|
+
"""A tool call chosen by the agent."""
|
|
35
|
+
|
|
36
|
+
tool: str
|
|
37
|
+
args: Mapping[str, Any] = field(default_factory=dict)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Observation:
|
|
42
|
+
"""What the robot reports back: text, camera images, and task status."""
|
|
43
|
+
|
|
44
|
+
text: str = ""
|
|
45
|
+
images: Mapping[str, bytes] = field(default_factory=dict) # camera name -> encoded image
|
|
46
|
+
done: bool = False
|
|
47
|
+
success: bool | None = None # None while the task is undecided
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@runtime_checkable
|
|
51
|
+
class Robot(Protocol):
|
|
52
|
+
"""The robot in a rua-bench task, as the agent under evaluation sees it."""
|
|
53
|
+
|
|
54
|
+
def tools(self) -> Sequence[Tool]: ...
|
|
55
|
+
|
|
56
|
+
def reset(self) -> Observation: ...
|
|
57
|
+
|
|
58
|
+
def step(self, action: Action) -> Observation: ...
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@runtime_checkable
|
|
62
|
+
class Agent(Protocol):
|
|
63
|
+
"""The agent under evaluation: picks the next action. Return None to stop."""
|
|
64
|
+
|
|
65
|
+
def act(self, observation: Observation, tools: Sequence[Tool]) -> Action | None: ...
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass(frozen=True)
|
|
69
|
+
class Step:
|
|
70
|
+
action: Action
|
|
71
|
+
observation: Observation
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass(frozen=True)
|
|
75
|
+
class Trajectory:
|
|
76
|
+
"""A finished episode: the first observation and every step after it."""
|
|
77
|
+
|
|
78
|
+
start: Observation
|
|
79
|
+
steps: tuple[Step, ...]
|
|
80
|
+
|
|
81
|
+
@property
|
|
82
|
+
def last(self) -> Observation:
|
|
83
|
+
return self.steps[-1].observation if self.steps else self.start
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def success(self) -> bool:
|
|
87
|
+
return self.last.success is True
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class ScriptedAgent:
|
|
91
|
+
"""Replays a fixed list of actions, then stops."""
|
|
92
|
+
|
|
93
|
+
def __init__(self, actions: Iterable[Action]) -> None:
|
|
94
|
+
self._actions = iter(tuple(actions))
|
|
95
|
+
|
|
96
|
+
def act(self, observation: Observation, tools: Sequence[Tool]) -> Action | None:
|
|
97
|
+
return next(self._actions, None)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def run(agent: Agent, robot: Robot, max_steps: int = 50) -> Trajectory:
|
|
101
|
+
"""Run one episode.
|
|
102
|
+
|
|
103
|
+
The episode ends when the robot reports done, the agent returns None, or
|
|
104
|
+
max_steps is reached. Raises ValueError if the agent calls a tool the
|
|
105
|
+
robot does not offer.
|
|
106
|
+
"""
|
|
107
|
+
tools = tuple(robot.tools())
|
|
108
|
+
names = {t.name for t in tools}
|
|
109
|
+
start = observation = robot.reset()
|
|
110
|
+
steps: list[Step] = []
|
|
111
|
+
while not observation.done and len(steps) < max_steps:
|
|
112
|
+
action = agent.act(observation, tools)
|
|
113
|
+
if action is None:
|
|
114
|
+
break
|
|
115
|
+
if action.tool not in names:
|
|
116
|
+
raise ValueError(f"unknown tool {action.tool!r}; robot offers {sorted(names)}")
|
|
117
|
+
observation = robot.step(action)
|
|
118
|
+
steps.append(Step(action, observation))
|
|
119
|
+
return Trajectory(start, tuple(steps))
|
|
File without changes
|