rua 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
rua-0.0.1/PKG-INFO ADDED
@@ -0,0 +1,75 @@
1
+ Metadata-Version: 2.3
2
+ Name: rua
3
+ Version: 0.0.1
4
+ Summary: The eval interface for rua-bench: how an agent and a benchmark task talk to each other, and the loop that runs one episode.
5
+ Keywords: rua-bench,benchmark,evaluation,robotics,agents
6
+ Classifier: Development Status :: 2 - Pre-Alpha
7
+ Classifier: Intended Audience :: Science/Research
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
10
+ Classifier: Typing :: Typed
11
+ Requires-Python: >=3.10
12
+ Description-Content-Type: text/markdown
13
+
14
+ # rua
15
+
16
+ The eval interface for [rua-bench](https://pypi.org/project/rua-bench/), a benchmark for robot-use agents (RUA): LLM agents that control a robot through tool calls.
17
+
18
+ `rua` is a helper for rua-bench. It defines how the agent under evaluation and a benchmark task talk to each other: the task's robot exposes tools and returns observations, the agent picks the next tool call, and `run` plays one episode. rua-bench then scores the episodes across a suite.
19
+
20
+ This is an early release. It defines the interface and the episode loop; the rua-bench tasks and model-backed agents will follow.
21
+
22
+ ## Install
23
+
24
+ ```bash
25
+ pip install rua
26
+ ```
27
+
28
+ ## Use
29
+
30
+ A task implements `Robot`:
31
+
32
+ ```python
33
+ from rua import Action, Observation, Tool
34
+
35
+ class Slider:
36
+ """A one-axis robot that must reach x = 3."""
37
+
38
+ def tools(self):
39
+ return [Tool("move_by", "Move along x", {"type": "object", "properties": {"dx": {"type": "number"}}})]
40
+
41
+ def reset(self):
42
+ self.x = 0.0
43
+ return Observation(text="x=0.0")
44
+
45
+ def step(self, action: Action):
46
+ self.x += action.args["dx"]
47
+ reached = self.x == 3.0
48
+ return Observation(text=f"x={self.x}", done=reached, success=reached or None)
49
+ ```
50
+
51
+ Run an agent against it. `ScriptedAgent` replays fixed actions; any object with an `act(observation, tools)` method works, including one that calls an LLM:
52
+
53
+ ```python
54
+ from rua import ScriptedAgent, run
55
+
56
+ agent = ScriptedAgent([Action("move_by", {"dx": 1.0}), Action("move_by", {"dx": 2.0})])
57
+ trajectory = run(agent, Slider())
58
+
59
+ trajectory.success # True
60
+ len(trajectory.steps) # 2
61
+ trajectory.last.text # "x=3.0"
62
+ ```
63
+
64
+ An episode ends when the robot reports `done`, the agent returns `None`, or `max_steps` is reached. `run` raises `ValueError` if the agent calls a tool the robot does not offer.
65
+
66
+ ## Scoring with rua-bench
67
+
68
+ Each trajectory becomes one rua-bench `Episode`:
69
+
70
+ ```python
71
+ from rua_bench import Episode, Suite, score
72
+
73
+ suite = Suite("demo", ("slider",))
74
+ print(score(suite, [Episode("slider", trajectory.success)]))
75
+ ```
rua-0.0.1/README.md ADDED
@@ -0,0 +1,62 @@
1
+ # rua
2
+
3
+ The eval interface for [rua-bench](https://pypi.org/project/rua-bench/), a benchmark for robot-use agents (RUA): LLM agents that control a robot through tool calls.
4
+
5
+ `rua` is a helper for rua-bench. It defines how the agent under evaluation and a benchmark task talk to each other: the task's robot exposes tools and returns observations, the agent picks the next tool call, and `run` plays one episode. rua-bench then scores the episodes across a suite.
6
+
7
+ This is an early release. It defines the interface and the episode loop; the rua-bench tasks and model-backed agents will follow.
8
+
9
+ ## Install
10
+
11
+ ```bash
12
+ pip install rua
13
+ ```
14
+
15
+ ## Use
16
+
17
+ A task implements `Robot`:
18
+
19
+ ```python
20
+ from rua import Action, Observation, Tool
21
+
22
+ class Slider:
23
+ """A one-axis robot that must reach x = 3."""
24
+
25
+ def tools(self):
26
+ return [Tool("move_by", "Move along x", {"type": "object", "properties": {"dx": {"type": "number"}}})]
27
+
28
+ def reset(self):
29
+ self.x = 0.0
30
+ return Observation(text="x=0.0")
31
+
32
+ def step(self, action: Action):
33
+ self.x += action.args["dx"]
34
+ reached = self.x == 3.0
35
+ return Observation(text=f"x={self.x}", done=reached, success=reached or None)
36
+ ```
37
+
38
+ Run an agent against it. `ScriptedAgent` replays fixed actions; any object with an `act(observation, tools)` method works, including one that calls an LLM:
39
+
40
+ ```python
41
+ from rua import ScriptedAgent, run
42
+
43
+ agent = ScriptedAgent([Action("move_by", {"dx": 1.0}), Action("move_by", {"dx": 2.0})])
44
+ trajectory = run(agent, Slider())
45
+
46
+ trajectory.success # True
47
+ len(trajectory.steps) # 2
48
+ trajectory.last.text # "x=3.0"
49
+ ```
50
+
51
+ An episode ends when the robot reports `done`, the agent returns `None`, or `max_steps` is reached. `run` raises `ValueError` if the agent calls a tool the robot does not offer.
52
+
53
+ ## Scoring with rua-bench
54
+
55
+ Each trajectory becomes one rua-bench `Episode`:
56
+
57
+ ```python
58
+ from rua_bench import Episode, Suite, score
59
+
60
+ suite = Suite("demo", ("slider",))
61
+ print(score(suite, [Episode("slider", trajectory.success)]))
62
+ ```
@@ -0,0 +1,25 @@
1
+ [project]
2
+ name = "rua"
3
+ version = "0.0.1"
4
+ description = "The eval interface for rua-bench: how an agent and a benchmark task talk to each other, and the loop that runs one episode."
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ dependencies = []
8
+ keywords = [
9
+ "rua-bench",
10
+ "benchmark",
11
+ "evaluation",
12
+ "robotics",
13
+ "agents",
14
+ ]
15
+ classifiers = [
16
+ "Development Status :: 2 - Pre-Alpha",
17
+ "Intended Audience :: Science/Research",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
20
+ "Typing :: Typed",
21
+ ]
22
+
23
+ [build-system]
24
+ requires = ["uv_build>=0.13.6,<0.14.0"]
25
+ build-backend = "uv_build"
@@ -0,0 +1,19 @@
1
+ [project]
2
+ name = "rua"
3
+ version = "0.0.1"
4
+ description = "The eval interface for rua-bench: how an agent and a benchmark task talk to each other, and the loop that runs one episode."
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ dependencies = []
8
+ keywords = ["rua-bench", "benchmark", "evaluation", "robotics", "agents"]
9
+ classifiers = [
10
+ "Development Status :: 2 - Pre-Alpha",
11
+ "Intended Audience :: Science/Research",
12
+ "Programming Language :: Python :: 3",
13
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
14
+ "Typing :: Typed",
15
+ ]
16
+
17
+ [build-system]
18
+ requires = ["uv_build>=0.13.6,<0.14.0"]
19
+ build-backend = "uv_build"
@@ -0,0 +1,119 @@
1
+ """The eval interface for rua-bench: how an agent and a benchmark task talk to each other."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Iterable, Mapping, Sequence
6
+ from dataclasses import dataclass, field
7
+ from typing import Any, Protocol, runtime_checkable
8
+
9
+ __version__ = "0.0.1"
10
+ __all__ = [
11
+ "Action",
12
+ "Agent",
13
+ "Observation",
14
+ "Robot",
15
+ "ScriptedAgent",
16
+ "Step",
17
+ "Tool",
18
+ "Trajectory",
19
+ "run",
20
+ ]
21
+
22
+
23
+ @dataclass(frozen=True)
24
+ class Tool:
25
+ """One call a robot accepts, described the way LLM tool APIs expect."""
26
+
27
+ name: str
28
+ description: str = ""
29
+ parameters: Mapping[str, Any] = field(default_factory=dict) # JSON Schema
30
+
31
+
32
+ @dataclass(frozen=True)
33
+ class Action:
34
+ """A tool call chosen by the agent."""
35
+
36
+ tool: str
37
+ args: Mapping[str, Any] = field(default_factory=dict)
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class Observation:
42
+ """What the robot reports back: text, camera images, and task status."""
43
+
44
+ text: str = ""
45
+ images: Mapping[str, bytes] = field(default_factory=dict) # camera name -> encoded image
46
+ done: bool = False
47
+ success: bool | None = None # None while the task is undecided
48
+
49
+
50
+ @runtime_checkable
51
+ class Robot(Protocol):
52
+ """The robot in a rua-bench task, as the agent under evaluation sees it."""
53
+
54
+ def tools(self) -> Sequence[Tool]: ...
55
+
56
+ def reset(self) -> Observation: ...
57
+
58
+ def step(self, action: Action) -> Observation: ...
59
+
60
+
61
+ @runtime_checkable
62
+ class Agent(Protocol):
63
+ """The agent under evaluation: picks the next action. Return None to stop."""
64
+
65
+ def act(self, observation: Observation, tools: Sequence[Tool]) -> Action | None: ...
66
+
67
+
68
+ @dataclass(frozen=True)
69
+ class Step:
70
+ action: Action
71
+ observation: Observation
72
+
73
+
74
+ @dataclass(frozen=True)
75
+ class Trajectory:
76
+ """A finished episode: the first observation and every step after it."""
77
+
78
+ start: Observation
79
+ steps: tuple[Step, ...]
80
+
81
+ @property
82
+ def last(self) -> Observation:
83
+ return self.steps[-1].observation if self.steps else self.start
84
+
85
+ @property
86
+ def success(self) -> bool:
87
+ return self.last.success is True
88
+
89
+
90
+ class ScriptedAgent:
91
+ """Replays a fixed list of actions, then stops."""
92
+
93
+ def __init__(self, actions: Iterable[Action]) -> None:
94
+ self._actions = iter(tuple(actions))
95
+
96
+ def act(self, observation: Observation, tools: Sequence[Tool]) -> Action | None:
97
+ return next(self._actions, None)
98
+
99
+
100
+ def run(agent: Agent, robot: Robot, max_steps: int = 50) -> Trajectory:
101
+ """Run one episode.
102
+
103
+ The episode ends when the robot reports done, the agent returns None, or
104
+ max_steps is reached. Raises ValueError if the agent calls a tool the
105
+ robot does not offer.
106
+ """
107
+ tools = tuple(robot.tools())
108
+ names = {t.name for t in tools}
109
+ start = observation = robot.reset()
110
+ steps: list[Step] = []
111
+ while not observation.done and len(steps) < max_steps:
112
+ action = agent.act(observation, tools)
113
+ if action is None:
114
+ break
115
+ if action.tool not in names:
116
+ raise ValueError(f"unknown tool {action.tool!r}; robot offers {sorted(names)}")
117
+ observation = robot.step(action)
118
+ steps.append(Step(action, observation))
119
+ return Trajectory(start, tuple(steps))
File without changes