abstract-client 0.2.2__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {abstract_client-0.2.2 → abstract_client-0.3.0}/NOTICE +2 -2
- abstract_client-0.2.2/README.md → abstract_client-0.3.0/PKG-INFO +132 -40
- abstract_client-0.2.2/PKG-INFO → abstract_client-0.3.0/README.md +107 -51
- abstract_client-0.3.0/pyproject.toml +39 -0
- abstract_client-0.3.0/src/abstract/__init__.py +109 -0
- {abstract_client-0.2.2/src/atoms → abstract_client-0.3.0/src/abstract}/errors.py +5 -1
- {abstract_client-0.2.2/src/atoms → abstract_client-0.3.0/src/abstract}/harness.py +73 -18
- {abstract_client-0.2.2/src/atoms → abstract_client-0.3.0/src/abstract}/policy.py +17 -11
- abstract_client-0.3.0/src/abstract/quic.py +360 -0
- abstract_client-0.2.2/src/atoms/client.py → abstract_client-0.3.0/src/abstract/transport.py +189 -118
- abstract_client-0.3.0/src/atoms/__init__.py +18 -0
- abstract_client-0.2.2/pyproject.toml +0 -19
- abstract_client-0.2.2/src/abstract/__init__.py +0 -43
- abstract_client-0.2.2/src/atoms/__init__.py +0 -70
- abstract_client-0.2.2/src/atoms/py.typed +0 -0
- {abstract_client-0.2.2 → abstract_client-0.3.0}/.gitignore +0 -0
- {abstract_client-0.2.2 → abstract_client-0.3.0}/LICENSE +0 -0
- {abstract_client-0.2.2 → abstract_client-0.3.0}/src/abstract/py.typed +0 -0
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
abstract-client (Abstract's robot API from Python, import name
|
|
1
|
+
abstract-client (Abstract's robot API from Python, import name abstract)
|
|
2
2
|
Copyright 2026 Abstract
|
|
3
3
|
|
|
4
4
|
Licensed under the Apache License, Version 2.0 (LICENSE).
|
|
5
5
|
|
|
6
|
-
The harness (
|
|
6
|
+
The harness (abstract.RobotClient) reproduces the calls of dreamlabs-client's DreamClient (policy,
|
|
7
7
|
set_observation, get_action, reset_episode, refill_due, close, last_timing), and its frame
|
|
8
8
|
window, refill pacing and slow-network warning are ported from dreamlabs-client
|
|
9
9
|
(https://github.com/dreamlabs-co/dreamlabs), first through the TypeScript harness
|
|
@@ -1,4 +1,29 @@
|
|
|
1
|
-
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: abstract-client
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Abstract's robot API from Python: action chunks from served robot models, sync and asyncio.
|
|
5
|
+
Project-URL: Homepage, https://abstract.inc
|
|
6
|
+
Project-URL: Documentation, https://docs.abstract.inc
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
License-File: NOTICE
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Typing :: Typed
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Requires-Dist: httpx2<3,>=2.12
|
|
22
|
+
Provides-Extra: quic
|
|
23
|
+
Requires-Dist: aioquic<2,>=1.3; extra == 'quic'
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# abstract.client
|
|
2
27
|
|
|
3
28
|
Abstract's robot API from Python. A `Client` holds your key and its connections and is safe to
|
|
4
29
|
share between threads; each `Policy` is one episode of one served robot model doing one task, one
|
|
@@ -8,12 +33,12 @@ action chunk per `predict()`.
|
|
|
8
33
|
uv add abstract-client
|
|
9
34
|
```
|
|
10
35
|
|
|
11
|
-
### Quickstart (0.
|
|
36
|
+
### Quickstart (0.3.0)
|
|
12
37
|
|
|
13
38
|
```python
|
|
14
39
|
import abstract
|
|
15
40
|
|
|
16
|
-
checkpoint = "
|
|
41
|
+
checkpoint = "allenai/molmoact2"
|
|
17
42
|
embodiment = "yam"
|
|
18
43
|
client = abstract.client(ABSTRACT_API_KEY, checkpoint, embodiment)
|
|
19
44
|
|
|
@@ -28,24 +53,27 @@ holds `prompt` (the task) and `state` (measured alongside the frames). The YAM c
|
|
|
28
53
|
state fields are shown below. The returned `Chunk` contains targets, never hardware motion;
|
|
29
54
|
reset ends the model's episode without repositioning the robot.
|
|
30
55
|
|
|
31
|
-
`abstract.client()` checks the account's unmetered model catalogue before configuring
|
|
32
|
-
`
|
|
33
|
-
|
|
56
|
+
`abstract.client()` checks the account's unmetered model catalogue before configuring a
|
|
57
|
+
`RobotClient`. The checkpoint must be available and its `robot` must match `embodiment`.
|
|
58
|
+
An optional URDF layer converts poses to another arm's joint geometry; it does not establish
|
|
59
|
+
policy transfer. Omit `api_key` to use the environment or saved
|
|
34
60
|
CLI credentials; connection options such as `base_url` and `timeout` pass through to the
|
|
35
|
-
existing client. Context managers and the
|
|
61
|
+
existing client. Context managers and the lower-level `abstract.Client` interface remain supported.
|
|
36
62
|
|
|
37
|
-
|
|
38
|
-
|
|
63
|
+
The `abstract.client(...)` factory has existed since 0.2.2. Version 0.3.0 moves the full SDK
|
|
64
|
+
into `abstract`, including the lower-level clients and policy types. Publish that version before
|
|
65
|
+
advertising the renamed advanced interfaces. `import atoms`, the 0.2 name, still resolves in 0.3
|
|
66
|
+
with a `DeprecationWarning`: its top-level names only, `AtomsClient` being `abstract.RobotClient`.
|
|
39
67
|
|
|
40
68
|
This is all a robot with its own stack needs. The `abstract` command and the robot station are
|
|
41
|
-
`abstract-
|
|
69
|
+
in `abstract-atoms`, built on it (`apps/atoms`).
|
|
42
70
|
|
|
43
71
|
```python
|
|
44
|
-
import
|
|
72
|
+
import abstract
|
|
45
73
|
|
|
46
|
-
client =
|
|
74
|
+
client = abstract.Client() # the key: api_key=, else ABSTRACT_API_KEY, else `abstract login`'s file
|
|
47
75
|
|
|
48
|
-
with client.connect("
|
|
76
|
+
with client.connect("allenai/molmoact2", "fold the towel") as policy:
|
|
49
77
|
frames = {"overhead": top_jpeg, "wrist_l": left_jpeg, "wrist_r": right_jpeg}
|
|
50
78
|
while not done:
|
|
51
79
|
state = {
|
|
@@ -58,15 +86,16 @@ with client.connect("nvidia/dreamzero", "fold the towel") as policy:
|
|
|
58
86
|
}
|
|
59
87
|
chunk = policy.predict(frames, state)
|
|
60
88
|
# Row k is a pose for t + k * chunk.dt; your arm's inverse kinematics turns each into
|
|
61
|
-
# joint angles.
|
|
62
|
-
|
|
89
|
+
# joint angles. The arm held still while predict() ran, so every row runs; a loop that
|
|
90
|
+
# keeps the arm moving meanwhile skips the rows it ran, as the RobotClient loop below.
|
|
91
|
+
for left, right in zip(chunk.left, chunk.right):
|
|
63
92
|
robot.command(left, right)
|
|
64
93
|
# The next call sends policy.frames_needed JPEGs per camera, oldest first.
|
|
65
94
|
frames = latest_frames(policy.frames_needed)
|
|
66
95
|
# Leaving the block ends the episode now; the server would after 60 s idle.
|
|
67
96
|
```
|
|
68
97
|
|
|
69
|
-
A stateful model
|
|
98
|
+
A stateful model takes the video since its last observation: after the first call,
|
|
70
99
|
send `policy.frames_needed` frames per camera.
|
|
71
100
|
|
|
72
101
|
### Native robot checkpoints
|
|
@@ -79,57 +108,63 @@ is enabled on the account. These native contracts are separate from the YAM exam
|
|
|
79
108
|
| Model | Observation | Returned commands |
|
|
80
109
|
| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
81
110
|
| `verne/v0.5` | The YAM example's poses and cameras, one frame per camera and then four; not trained on the YAM | 18 pose rows per arm at 30 Hz |
|
|
111
|
+
| `runway/praxis-1` | The YAM example's poses and cameras at 16:10, one frame per camera and then 21: one at the end of each row just run. Capture them yourself and call `policy.predict()`; `RobotClient`'s window also counts the pause between chunks. | 21 pose rows per arm at 15 Hz |
|
|
82
112
|
| `xiaomi/xiaomi-robotics-1` | RoboDojo ARX X5 world-frame poses, normalized grippers (0 closed, 1 open), and `left_joint_position` / `right_joint_position` with six measured radians each. Cameras retain `overhead`, `wrist_l`, `wrist_r`. | `chunk.left` / `chunk.right`: 30 end-effector targets at 25 Hz, in the same RoboDojo world frame. |
|
|
83
113
|
| `black-forest-labs/flux-3-action` | DROID Franka `joint_position` (seven radians), `gripper_position` (one closed fraction: 0 open, 1 closed), and cameras `exterior_image_0_left`, `exterior_image_1_left`, `wrist_image_left`. | `chunk.joints`: 32 rows of seven absolute joint targets plus gripper at 15 Hz; `chunk.action_space == "joint_position"`. `left` and `right` are empty. |
|
|
84
114
|
|
|
85
115
|
Use `chunk.row_count` for either command format. `elapsed_rows`, sync/async policies, retries
|
|
86
|
-
and the `
|
|
87
|
-
window must use the same state and camera contract. The
|
|
116
|
+
and the `RobotClient` observation window work with both. Policies sharing one observation
|
|
117
|
+
window must use the same state and camera contract. The bimanual station does not drive these native contracts.
|
|
88
118
|
|
|
89
|
-
### `
|
|
119
|
+
### `abstract.RobotClient`: Dream Labs' DreamClient on this API
|
|
90
120
|
|
|
91
|
-
`
|
|
121
|
+
`abstract.RobotClient` keeps that window for you and paces the requests, with the calls of Dream Labs'
|
|
92
122
|
`DreamClient` (Apache-2.0; see `NOTICE`): bind the key, choose the model, push every camera tick,
|
|
93
123
|
pull a chunk when it says so.
|
|
94
124
|
|
|
95
125
|
```python
|
|
96
126
|
from concurrent.futures import ThreadPoolExecutor
|
|
97
127
|
|
|
98
|
-
robot =
|
|
99
|
-
robot.policy({"model": "
|
|
128
|
+
robot = abstract.RobotClient() # the key, as abstract.Client finds it
|
|
129
|
+
robot.policy({"model": "allenai/molmoact2"})
|
|
100
130
|
with robot, ThreadPoolExecutor(1) as worker: # the request runs while the robot moves
|
|
101
131
|
chunk, row, pending = None, 0, None
|
|
132
|
+
ran = ran_at_ask = 0 # rows the robot has run, and how many when the request went out
|
|
102
133
|
while not done:
|
|
103
134
|
frames = {"video.overhead": top, "video.wrist_l": left, "video.wrist_r": right}
|
|
104
135
|
robot.set_observation(frames, {"prompt": "fold the towel", "state": read_state()})
|
|
105
136
|
rows_left = len(chunk.left) - row if chunk else 0
|
|
106
137
|
if pending is None and robot.refill_due(row, rows_left):
|
|
107
|
-
pending = worker.submit(robot.get_action)
|
|
138
|
+
pending, ran_at_ask = worker.submit(robot.get_action), ran
|
|
108
139
|
if pending is not None and pending.done():
|
|
109
140
|
chunk, pending = pending.result(), None
|
|
110
|
-
|
|
141
|
+
# Skip the rows the robot ran while this chunk was computed (at most those already
|
|
142
|
+
# due, chunk.elapsed_rows). A robot that held still meanwhile starts at row 0.
|
|
143
|
+
row = min(chunk.elapsed_rows, ran - ran_at_ask)
|
|
111
144
|
if chunk and row < len(chunk.left):
|
|
112
145
|
robot.command(chunk.left[row], chunk.right[row])
|
|
113
|
-
row
|
|
146
|
+
row, ran = row + 1, ran + 1
|
|
114
147
|
wait_for_next_tick()
|
|
115
148
|
```
|
|
116
149
|
|
|
117
150
|
#### Several models on one robot
|
|
118
151
|
|
|
119
|
-
DreamClient runs one model. `
|
|
120
|
-
with its own episode, frames (
|
|
152
|
+
DreamClient runs one model. `RobotClient` runs as many as you open on the same camera ticks, each
|
|
153
|
+
with its own episode, frames (a stateful model takes several per camera, MolmoAct2 one), pacing and timing.
|
|
121
154
|
One drives the arms; the others answer the same observations, to compare them.
|
|
122
155
|
|
|
123
156
|
```python
|
|
124
|
-
robot.policies(
|
|
157
|
+
robot.policies(
|
|
158
|
+
{"molmo": {"model": "allenai/molmoact2"}, "pi": {"model": "physical-intelligence/pi0.5"}}
|
|
159
|
+
)
|
|
125
160
|
robot.set_observation(frames, observation) # one tick, every model
|
|
126
161
|
answers = robot.get_actions() # every model at once: {name: Chunk, or the error it raised}
|
|
127
|
-
drive = robot["
|
|
162
|
+
drive = robot["molmo"] # one model: get_action(), refill_due(), last, mode, last_timing
|
|
128
163
|
```
|
|
129
164
|
|
|
130
|
-
| DreamClient |
|
|
165
|
+
| DreamClient | abstract.RobotClient |
|
|
131
166
|
| ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
|
132
|
-
| `DreamClient(api_key)` | `
|
|
167
|
+
| `DreamClient(api_key)` | `RobotClient(api_key)`, or `RobotClient(client=...)` |
|
|
133
168
|
| `get_models()` | `get_models()`: served models and your fine-tunes, with their rates |
|
|
134
169
|
| `policy({"checkpoint": ...})` | `policy({"model": ...})`; `execution_horizon`, `action_hz`, `sync_above_ms`, `timeout_ms` as in DreamClient |
|
|
135
170
|
| `set_observation(frames, observation, timestamp=)` | the same: `video.<camera>` frames, `prompt` and `state` (a dict or `state.<name>` keys) |
|
|
@@ -137,13 +172,35 @@ drive = robot["dz"] # one model: get_action(), refill_due(), last, mode, last_t
|
|
|
137
172
|
| `reset_episode()`, `refill_due()`, `close()` | the same |
|
|
138
173
|
| `mode`, `execution_horizon`, `action_hz`, `last_timing`, `checkpoint`, `session_id` | the same, `model` for `checkpoint` |
|
|
139
174
|
| `obs_window`, `obs_fps` | none: each answer says how many frames the next observation carries |
|
|
140
|
-
| `DreamSlowNetworkWarning` | `
|
|
175
|
+
| `DreamSlowNetworkWarning` | `abstract.SlowNetworkWarning` |
|
|
141
176
|
| one model per client | `policies({name: config})`, `get_actions()`, `robot[name]` |
|
|
142
177
|
|
|
143
|
-
|
|
144
|
-
|
|
178
|
+
### Optional URDF FK and IK (0.3.0)
|
|
179
|
+
|
|
180
|
+
The solver is not in this package: install `abstract-atoms[ik]` after release, or
|
|
181
|
+
`uv pip install -e ./apps/atoms/client -e './apps/atoms[ik]'` from this checkout. Configure each physical arm's URDF, joint order, grasp point and frames:
|
|
182
|
+
|
|
183
|
+
```python
|
|
184
|
+
from abstract_robots.ik import RobotIK
|
|
185
|
+
|
|
186
|
+
ik = RobotIK.from_file("robot.json")
|
|
187
|
+
client = abstract.client(checkpoint="allenai/molmoact2", embodiment="yam", ik=ik)
|
|
188
|
+
client.set_joint_observation(frames, measured_joints, grippers, prompt="fold the towel")
|
|
189
|
+
chunk = client.get_action()
|
|
190
|
+
# Measure again after inference; solving is local and commands no hardware. The arm held still
|
|
191
|
+
# during this call; one that kept moving passes start= the rows it ran (at most elapsed_rows).
|
|
192
|
+
joints = client.to_joints(chunk, current_measured_joints)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
See [the configuration and driver guide](../robots/IK.md) for full examples, limits, geometry checks and
|
|
196
|
+
validation scope. Different arm geometries/joint counts are supported locally. Model embodiment
|
|
197
|
+
checks remain; correct IK does not prove policy transfer. The Python station uses the same solver
|
|
198
|
+
through `--ik robot.json`, with `observe_joints()` / `command_joints()` drivers.
|
|
199
|
+
|
|
200
|
+
DreamClient's seam smoothing (`commanded`) is not implemented. Dataset and fine-tuning commands
|
|
201
|
+
and the station's Live View remain in `abstract-atoms`.
|
|
145
202
|
|
|
146
|
-
`
|
|
203
|
+
`abstract.AsyncClient` is native asyncio with the same arguments, rules and calls
|
|
147
204
|
(`await policy.predict(...)`). Cancelling a call cancels its request, but the server may already
|
|
148
205
|
be running it, so the next `predict()` asks for that answer first, as after a lost connection.
|
|
149
206
|
|
|
@@ -159,10 +216,41 @@ import httpx2
|
|
|
159
216
|
|
|
160
217
|
tls = ssl.create_default_context(cafile="lab-ca.pem")
|
|
161
218
|
tls.load_cert_chain("robot.pem", "robot.key")
|
|
162
|
-
client =
|
|
219
|
+
client = abstract.Client(http_client=httpx2.Client(verify=tls, proxy="http://proxy.lab:3128"))
|
|
163
220
|
```
|
|
164
221
|
|
|
165
|
-
|
|
222
|
+
### Transports
|
|
223
|
+
|
|
224
|
+
`transport` chooses how the calls travel. `"http"` is the default and what everything above
|
|
225
|
+
describes. `"quic"` sends the same calls to the same routes over HTTP/3, on one QUIC connection
|
|
226
|
+
that every policy on the client shares; it is opt-in and needs the `quic` extra:
|
|
227
|
+
|
|
228
|
+
```sh
|
|
229
|
+
uv add 'abstract-client[quic]'
|
|
230
|
+
```
|
|
231
|
+
|
|
232
|
+
```python
|
|
233
|
+
client = abstract.Client(transport="quic") # or ABSTRACT_TRANSPORT=quic; every client takes it
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
The API sees the same request either way, so chunks are billed the same and a call that gets no
|
|
237
|
+
answer is sent again under the same rules. What differs is the network's side:
|
|
238
|
+
|
|
239
|
+
- QUIC needs outbound UDP to the API on port 443. Where a network blocks it, calls fail with
|
|
240
|
+
`APIConnectionError` or `APITimeoutError`; nothing switches to `"http"` on its own.
|
|
241
|
+
- No proxy can carry it. On a machine set to use one (`HTTPS_PROXY`, `ALL_PROXY` or the system's)
|
|
242
|
+
the client raises `ConfigError` rather than go around the proxy, and `http_client=` cannot be
|
|
243
|
+
combined with it.
|
|
244
|
+
- TLS is checked against certifi's CA bundle, not the operating system's trust store: a CA your
|
|
245
|
+
IT installed is not trusted, and there is no setting for a private CA or a client certificate.
|
|
246
|
+
- A connection that hears nothing for 20 s is closed and the next call opens another. While a
|
|
247
|
+
call waits for its answer the client pings every 5 s, so a model's cold start does not end it.
|
|
248
|
+
|
|
249
|
+
It runs aioquic's QUIC stack as it comes, with no tuning, and has not been measured against
|
|
250
|
+
`"http"`: choose it to compare the two on your own network, not on a promise that it is faster.
|
|
251
|
+
`client.transport` says which one a client uses.
|
|
252
|
+
|
|
253
|
+
`connect(model, task, scaling={"seed": 7})` (or `scaling` in a `RobotClient` config) sends the
|
|
166
254
|
API's `scaling` with every observation, so a seedable model replays its noise; the engine's reports
|
|
167
255
|
(`scaling`, `cache`, `context`) come back in `chunk.raw`.
|
|
168
256
|
|
|
@@ -171,12 +259,16 @@ bytes and `observation_id`, so the server answers it from its record and never b
|
|
|
171
259
|
Refusals the server marks retryable are sent again after its `Retry-After`, up to `max_retries`;
|
|
172
260
|
`timeout` bounds a whole call, resends included, and `connect_timeout` opening a connection.
|
|
173
261
|
|
|
174
|
-
Every error is an `
|
|
262
|
+
Every error is an `abstract.AbstractError`. A refusal is an `abstract.APIStatusError` subclass named
|
|
175
263
|
for its status (`AuthenticationError`, `PaymentRequiredError`, `RateLimitError`, ...) with the
|
|
176
264
|
server's `code`, `retryable`, `retry_after` and `request_id`; a refused or answered
|
|
177
265
|
`get_action` also carries its reply's `trace_id`, which a replay keeps. `APIConnectionError` and
|
|
178
|
-
`APITimeoutError` mean no answer came: the next `predict()` asks for it first. The `
|
|
266
|
+
`APITimeoutError` mean no answer came: the next `predict()` asks for it first. The `abstract.client`
|
|
179
267
|
logger reports resends; it never logs the key.
|
|
180
268
|
|
|
181
|
-
Python 3.10 or later; its one dependency is `httpx2
|
|
269
|
+
Python 3.10 or later; its one dependency is `httpx2` (`aioquic` with the `quic` extra). Tests: `pytest apps/atoms` (the contract
|
|
182
270
|
tests need Bun and the engine's packages: `uv run --project apps/engine pytest apps/atoms`).
|
|
271
|
+
|
|
272
|
+
Version 0.3 replaces `import atoms` with `import abstract`; the buffered harness is
|
|
273
|
+
`abstract.RobotClient` (previously `AtomsClient`). `abstract.client(...)` is the recommended
|
|
274
|
+
entry point. The Python station uses this same implementation; there is no separate YAM runner.
|
|
@@ -1,15 +1,4 @@
|
|
|
1
|
-
|
|
2
|
-
Name: abstract-client
|
|
3
|
-
Version: 0.2.2
|
|
4
|
-
Summary: Abstract's robot API from Python: action chunks from served robot models, sync and asyncio.
|
|
5
|
-
License-Expression: Apache-2.0
|
|
6
|
-
License-File: LICENSE
|
|
7
|
-
License-File: NOTICE
|
|
8
|
-
Requires-Python: >=3.10
|
|
9
|
-
Requires-Dist: httpx2<3,>=2.12
|
|
10
|
-
Description-Content-Type: text/markdown
|
|
11
|
-
|
|
12
|
-
# abstract-client
|
|
1
|
+
# abstract.client
|
|
13
2
|
|
|
14
3
|
Abstract's robot API from Python. A `Client` holds your key and its connections and is safe to
|
|
15
4
|
share between threads; each `Policy` is one episode of one served robot model doing one task, one
|
|
@@ -19,12 +8,12 @@ action chunk per `predict()`.
|
|
|
19
8
|
uv add abstract-client
|
|
20
9
|
```
|
|
21
10
|
|
|
22
|
-
### Quickstart (0.
|
|
11
|
+
### Quickstart (0.3.0)
|
|
23
12
|
|
|
24
13
|
```python
|
|
25
14
|
import abstract
|
|
26
15
|
|
|
27
|
-
checkpoint = "
|
|
16
|
+
checkpoint = "allenai/molmoact2"
|
|
28
17
|
embodiment = "yam"
|
|
29
18
|
client = abstract.client(ABSTRACT_API_KEY, checkpoint, embodiment)
|
|
30
19
|
|
|
@@ -39,24 +28,27 @@ holds `prompt` (the task) and `state` (measured alongside the frames). The YAM c
|
|
|
39
28
|
state fields are shown below. The returned `Chunk` contains targets, never hardware motion;
|
|
40
29
|
reset ends the model's episode without repositioning the robot.
|
|
41
30
|
|
|
42
|
-
`abstract.client()` checks the account's unmetered model catalogue before configuring
|
|
43
|
-
`
|
|
44
|
-
|
|
31
|
+
`abstract.client()` checks the account's unmetered model catalogue before configuring a
|
|
32
|
+
`RobotClient`. The checkpoint must be available and its `robot` must match `embodiment`.
|
|
33
|
+
An optional URDF layer converts poses to another arm's joint geometry; it does not establish
|
|
34
|
+
policy transfer. Omit `api_key` to use the environment or saved
|
|
45
35
|
CLI credentials; connection options such as `base_url` and `timeout` pass through to the
|
|
46
|
-
existing client. Context managers and the
|
|
36
|
+
existing client. Context managers and the lower-level `abstract.Client` interface remain supported.
|
|
47
37
|
|
|
48
|
-
|
|
49
|
-
|
|
38
|
+
The `abstract.client(...)` factory has existed since 0.2.2. Version 0.3.0 moves the full SDK
|
|
39
|
+
into `abstract`, including the lower-level clients and policy types. Publish that version before
|
|
40
|
+
advertising the renamed advanced interfaces. `import atoms`, the 0.2 name, still resolves in 0.3
|
|
41
|
+
with a `DeprecationWarning`: its top-level names only, `AtomsClient` being `abstract.RobotClient`.
|
|
50
42
|
|
|
51
43
|
This is all a robot with its own stack needs. The `abstract` command and the robot station are
|
|
52
|
-
`abstract-
|
|
44
|
+
in `abstract-atoms`, built on it (`apps/atoms`).
|
|
53
45
|
|
|
54
46
|
```python
|
|
55
|
-
import
|
|
47
|
+
import abstract
|
|
56
48
|
|
|
57
|
-
client =
|
|
49
|
+
client = abstract.Client() # the key: api_key=, else ABSTRACT_API_KEY, else `abstract login`'s file
|
|
58
50
|
|
|
59
|
-
with client.connect("
|
|
51
|
+
with client.connect("allenai/molmoact2", "fold the towel") as policy:
|
|
60
52
|
frames = {"overhead": top_jpeg, "wrist_l": left_jpeg, "wrist_r": right_jpeg}
|
|
61
53
|
while not done:
|
|
62
54
|
state = {
|
|
@@ -69,15 +61,16 @@ with client.connect("nvidia/dreamzero", "fold the towel") as policy:
|
|
|
69
61
|
}
|
|
70
62
|
chunk = policy.predict(frames, state)
|
|
71
63
|
# Row k is a pose for t + k * chunk.dt; your arm's inverse kinematics turns each into
|
|
72
|
-
# joint angles.
|
|
73
|
-
|
|
64
|
+
# joint angles. The arm held still while predict() ran, so every row runs; a loop that
|
|
65
|
+
# keeps the arm moving meanwhile skips the rows it ran, as the RobotClient loop below.
|
|
66
|
+
for left, right in zip(chunk.left, chunk.right):
|
|
74
67
|
robot.command(left, right)
|
|
75
68
|
# The next call sends policy.frames_needed JPEGs per camera, oldest first.
|
|
76
69
|
frames = latest_frames(policy.frames_needed)
|
|
77
70
|
# Leaving the block ends the episode now; the server would after 60 s idle.
|
|
78
71
|
```
|
|
79
72
|
|
|
80
|
-
A stateful model
|
|
73
|
+
A stateful model takes the video since its last observation: after the first call,
|
|
81
74
|
send `policy.frames_needed` frames per camera.
|
|
82
75
|
|
|
83
76
|
### Native robot checkpoints
|
|
@@ -90,57 +83,63 @@ is enabled on the account. These native contracts are separate from the YAM exam
|
|
|
90
83
|
| Model | Observation | Returned commands |
|
|
91
84
|
| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
92
85
|
| `verne/v0.5` | The YAM example's poses and cameras, one frame per camera and then four; not trained on the YAM | 18 pose rows per arm at 30 Hz |
|
|
86
|
+
| `runway/praxis-1` | The YAM example's poses and cameras at 16:10, one frame per camera and then 21: one at the end of each row just run. Capture them yourself and call `policy.predict()`; `RobotClient`'s window also counts the pause between chunks. | 21 pose rows per arm at 15 Hz |
|
|
93
87
|
| `xiaomi/xiaomi-robotics-1` | RoboDojo ARX X5 world-frame poses, normalized grippers (0 closed, 1 open), and `left_joint_position` / `right_joint_position` with six measured radians each. Cameras retain `overhead`, `wrist_l`, `wrist_r`. | `chunk.left` / `chunk.right`: 30 end-effector targets at 25 Hz, in the same RoboDojo world frame. |
|
|
94
88
|
| `black-forest-labs/flux-3-action` | DROID Franka `joint_position` (seven radians), `gripper_position` (one closed fraction: 0 open, 1 closed), and cameras `exterior_image_0_left`, `exterior_image_1_left`, `wrist_image_left`. | `chunk.joints`: 32 rows of seven absolute joint targets plus gripper at 15 Hz; `chunk.action_space == "joint_position"`. `left` and `right` are empty. |
|
|
95
89
|
|
|
96
90
|
Use `chunk.row_count` for either command format. `elapsed_rows`, sync/async policies, retries
|
|
97
|
-
and the `
|
|
98
|
-
window must use the same state and camera contract. The
|
|
91
|
+
and the `RobotClient` observation window work with both. Policies sharing one observation
|
|
92
|
+
window must use the same state and camera contract. The bimanual station does not drive these native contracts.
|
|
99
93
|
|
|
100
|
-
### `
|
|
94
|
+
### `abstract.RobotClient`: Dream Labs' DreamClient on this API
|
|
101
95
|
|
|
102
|
-
`
|
|
96
|
+
`abstract.RobotClient` keeps that window for you and paces the requests, with the calls of Dream Labs'
|
|
103
97
|
`DreamClient` (Apache-2.0; see `NOTICE`): bind the key, choose the model, push every camera tick,
|
|
104
98
|
pull a chunk when it says so.
|
|
105
99
|
|
|
106
100
|
```python
|
|
107
101
|
from concurrent.futures import ThreadPoolExecutor
|
|
108
102
|
|
|
109
|
-
robot =
|
|
110
|
-
robot.policy({"model": "
|
|
103
|
+
robot = abstract.RobotClient() # the key, as abstract.Client finds it
|
|
104
|
+
robot.policy({"model": "allenai/molmoact2"})
|
|
111
105
|
with robot, ThreadPoolExecutor(1) as worker: # the request runs while the robot moves
|
|
112
106
|
chunk, row, pending = None, 0, None
|
|
107
|
+
ran = ran_at_ask = 0 # rows the robot has run, and how many when the request went out
|
|
113
108
|
while not done:
|
|
114
109
|
frames = {"video.overhead": top, "video.wrist_l": left, "video.wrist_r": right}
|
|
115
110
|
robot.set_observation(frames, {"prompt": "fold the towel", "state": read_state()})
|
|
116
111
|
rows_left = len(chunk.left) - row if chunk else 0
|
|
117
112
|
if pending is None and robot.refill_due(row, rows_left):
|
|
118
|
-
pending = worker.submit(robot.get_action)
|
|
113
|
+
pending, ran_at_ask = worker.submit(robot.get_action), ran
|
|
119
114
|
if pending is not None and pending.done():
|
|
120
115
|
chunk, pending = pending.result(), None
|
|
121
|
-
|
|
116
|
+
# Skip the rows the robot ran while this chunk was computed (at most those already
|
|
117
|
+
# due, chunk.elapsed_rows). A robot that held still meanwhile starts at row 0.
|
|
118
|
+
row = min(chunk.elapsed_rows, ran - ran_at_ask)
|
|
122
119
|
if chunk and row < len(chunk.left):
|
|
123
120
|
robot.command(chunk.left[row], chunk.right[row])
|
|
124
|
-
row
|
|
121
|
+
row, ran = row + 1, ran + 1
|
|
125
122
|
wait_for_next_tick()
|
|
126
123
|
```
|
|
127
124
|
|
|
128
125
|
#### Several models on one robot
|
|
129
126
|
|
|
130
|
-
DreamClient runs one model. `
|
|
131
|
-
with its own episode, frames (
|
|
127
|
+
DreamClient runs one model. `RobotClient` runs as many as you open on the same camera ticks, each
|
|
128
|
+
with its own episode, frames (a stateful model takes several per camera, MolmoAct2 one), pacing and timing.
|
|
132
129
|
One drives the arms; the others answer the same observations, to compare them.
|
|
133
130
|
|
|
134
131
|
```python
|
|
135
|
-
robot.policies(
|
|
132
|
+
robot.policies(
|
|
133
|
+
{"molmo": {"model": "allenai/molmoact2"}, "pi": {"model": "physical-intelligence/pi0.5"}}
|
|
134
|
+
)
|
|
136
135
|
robot.set_observation(frames, observation) # one tick, every model
|
|
137
136
|
answers = robot.get_actions() # every model at once: {name: Chunk, or the error it raised}
|
|
138
|
-
drive = robot["
|
|
137
|
+
drive = robot["molmo"] # one model: get_action(), refill_due(), last, mode, last_timing
|
|
139
138
|
```
|
|
140
139
|
|
|
141
|
-
| DreamClient |
|
|
140
|
+
| DreamClient | abstract.RobotClient |
|
|
142
141
|
| ----------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
|
143
|
-
| `DreamClient(api_key)` | `
|
|
142
|
+
| `DreamClient(api_key)` | `RobotClient(api_key)`, or `RobotClient(client=...)` |
|
|
144
143
|
| `get_models()` | `get_models()`: served models and your fine-tunes, with their rates |
|
|
145
144
|
| `policy({"checkpoint": ...})` | `policy({"model": ...})`; `execution_horizon`, `action_hz`, `sync_above_ms`, `timeout_ms` as in DreamClient |
|
|
146
145
|
| `set_observation(frames, observation, timestamp=)` | the same: `video.<camera>` frames, `prompt` and `state` (a dict or `state.<name>` keys) |
|
|
@@ -148,13 +147,35 @@ drive = robot["dz"] # one model: get_action(), refill_due(), last, mode, last_t
|
|
|
148
147
|
| `reset_episode()`, `refill_due()`, `close()` | the same |
|
|
149
148
|
| `mode`, `execution_horizon`, `action_hz`, `last_timing`, `checkpoint`, `session_id` | the same, `model` for `checkpoint` |
|
|
150
149
|
| `obs_window`, `obs_fps` | none: each answer says how many frames the next observation carries |
|
|
151
|
-
| `DreamSlowNetworkWarning` | `
|
|
150
|
+
| `DreamSlowNetworkWarning` | `abstract.SlowNetworkWarning` |
|
|
152
151
|
| one model per client | `policies({name: config})`, `get_actions()`, `robot[name]` |
|
|
153
152
|
|
|
154
|
-
|
|
155
|
-
|
|
153
|
+
### Optional URDF FK and IK (0.3.0)
|
|
154
|
+
|
|
155
|
+
The solver is not in this package: install `abstract-atoms[ik]` after release, or
|
|
156
|
+
`uv pip install -e ./apps/atoms/client -e './apps/atoms[ik]'` from this checkout. Configure each physical arm's URDF, joint order, grasp point and frames:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
from abstract_robots.ik import RobotIK
|
|
160
|
+
|
|
161
|
+
ik = RobotIK.from_file("robot.json")
|
|
162
|
+
client = abstract.client(checkpoint="allenai/molmoact2", embodiment="yam", ik=ik)
|
|
163
|
+
client.set_joint_observation(frames, measured_joints, grippers, prompt="fold the towel")
|
|
164
|
+
chunk = client.get_action()
|
|
165
|
+
# Measure again after inference; solving is local and commands no hardware. The arm held still
|
|
166
|
+
# during this call; one that kept moving passes start= the rows it ran (at most elapsed_rows).
|
|
167
|
+
joints = client.to_joints(chunk, current_measured_joints)
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
See [the configuration and driver guide](../robots/IK.md) for full examples, limits, geometry checks and
|
|
171
|
+
validation scope. Different arm geometries/joint counts are supported locally. Model embodiment
|
|
172
|
+
checks remain; correct IK does not prove policy transfer. The Python station uses the same solver
|
|
173
|
+
through `--ik robot.json`, with `observe_joints()` / `command_joints()` drivers.
|
|
174
|
+
|
|
175
|
+
DreamClient's seam smoothing (`commanded`) is not implemented. Dataset and fine-tuning commands
|
|
176
|
+
and the station's Live View remain in `abstract-atoms`.
|
|
156
177
|
|
|
157
|
-
`
|
|
178
|
+
`abstract.AsyncClient` is native asyncio with the same arguments, rules and calls
|
|
158
179
|
(`await policy.predict(...)`). Cancelling a call cancels its request, but the server may already
|
|
159
180
|
be running it, so the next `predict()` asks for that answer first, as after a lost connection.
|
|
160
181
|
|
|
@@ -170,10 +191,41 @@ import httpx2
|
|
|
170
191
|
|
|
171
192
|
tls = ssl.create_default_context(cafile="lab-ca.pem")
|
|
172
193
|
tls.load_cert_chain("robot.pem", "robot.key")
|
|
173
|
-
client =
|
|
194
|
+
client = abstract.Client(http_client=httpx2.Client(verify=tls, proxy="http://proxy.lab:3128"))
|
|
174
195
|
```
|
|
175
196
|
|
|
176
|
-
|
|
197
|
+
### Transports
|
|
198
|
+
|
|
199
|
+
`transport` chooses how the calls travel. `"http"` is the default and what everything above
|
|
200
|
+
describes. `"quic"` sends the same calls to the same routes over HTTP/3, on one QUIC connection
|
|
201
|
+
that every policy on the client shares; it is opt-in and needs the `quic` extra:
|
|
202
|
+
|
|
203
|
+
```sh
|
|
204
|
+
uv add 'abstract-client[quic]'
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
```python
|
|
208
|
+
client = abstract.Client(transport="quic") # or ABSTRACT_TRANSPORT=quic; every client takes it
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
The API sees the same request either way, so chunks are billed the same and a call that gets no
|
|
212
|
+
answer is sent again under the same rules. What differs is the network's side:
|
|
213
|
+
|
|
214
|
+
- QUIC needs outbound UDP to the API on port 443. Where a network blocks it, calls fail with
|
|
215
|
+
`APIConnectionError` or `APITimeoutError`; nothing switches to `"http"` on its own.
|
|
216
|
+
- No proxy can carry it. On a machine set to use one (`HTTPS_PROXY`, `ALL_PROXY` or the system's)
|
|
217
|
+
the client raises `ConfigError` rather than go around the proxy, and `http_client=` cannot be
|
|
218
|
+
combined with it.
|
|
219
|
+
- TLS is checked against certifi's CA bundle, not the operating system's trust store: a CA your
|
|
220
|
+
IT installed is not trusted, and there is no setting for a private CA or a client certificate.
|
|
221
|
+
- A connection that hears nothing for 20 s is closed and the next call opens another. While a
|
|
222
|
+
call waits for its answer the client pings every 5 s, so a model's cold start does not end it.
|
|
223
|
+
|
|
224
|
+
It runs aioquic's QUIC stack as it comes, with no tuning, and has not been measured against
|
|
225
|
+
`"http"`: choose it to compare the two on your own network, not on a promise that it is faster.
|
|
226
|
+
`client.transport` says which one a client uses.
|
|
227
|
+
|
|
228
|
+
`connect(model, task, scaling={"seed": 7})` (or `scaling` in a `RobotClient` config) sends the
|
|
177
229
|
API's `scaling` with every observation, so a seedable model replays its noise; the engine's reports
|
|
178
230
|
(`scaling`, `cache`, `context`) come back in `chunk.raw`.
|
|
179
231
|
|
|
@@ -182,12 +234,16 @@ bytes and `observation_id`, so the server answers it from its record and never b
|
|
|
182
234
|
Refusals the server marks retryable are sent again after its `Retry-After`, up to `max_retries`;
|
|
183
235
|
`timeout` bounds a whole call, resends included, and `connect_timeout` opening a connection.
|
|
184
236
|
|
|
185
|
-
Every error is an `
|
|
237
|
+
Every error is an `abstract.AbstractError`. A refusal is an `abstract.APIStatusError` subclass named
|
|
186
238
|
for its status (`AuthenticationError`, `PaymentRequiredError`, `RateLimitError`, ...) with the
|
|
187
239
|
server's `code`, `retryable`, `retry_after` and `request_id`; a refused or answered
|
|
188
240
|
`get_action` also carries its reply's `trace_id`, which a replay keeps. `APIConnectionError` and
|
|
189
|
-
`APITimeoutError` mean no answer came: the next `predict()` asks for it first. The `
|
|
241
|
+
`APITimeoutError` mean no answer came: the next `predict()` asks for it first. The `abstract.client`
|
|
190
242
|
logger reports resends; it never logs the key.
|
|
191
243
|
|
|
192
|
-
Python 3.10 or later; its one dependency is `httpx2
|
|
244
|
+
Python 3.10 or later; its one dependency is `httpx2` (`aioquic` with the `quic` extra). Tests: `pytest apps/atoms` (the contract
|
|
193
245
|
tests need Bun and the engine's packages: `uv run --project apps/engine pytest apps/atoms`).
|
|
246
|
+
|
|
247
|
+
Version 0.3 replaces `import atoms` with `import abstract`; the buffered harness is
|
|
248
|
+
`abstract.RobotClient` (previously `AtomsClient`). `abstract.client(...)` is the recommended
|
|
249
|
+
entry point. The Python station uses this same implementation; there is no separate YAM runner.
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "abstract-client"
|
|
3
|
+
# The one place this wheel's version is written: `abstract.__version__` reads it from the
|
|
4
|
+
# installed wheel. Raise it here to release this wheel; abstract-atoms accepts any 0.3.
|
|
5
|
+
version = "0.3.0"
|
|
6
|
+
description = "Abstract's robot API from Python: action chunks from served robot models, sync and asyncio."
|
|
7
|
+
readme = "README.md"
|
|
8
|
+
license = "Apache-2.0"
|
|
9
|
+
requires-python = ">=3.10"
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Intended Audience :: Developers",
|
|
12
|
+
"Intended Audience :: Science/Research",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.10",
|
|
15
|
+
"Programming Language :: Python :: 3.11",
|
|
16
|
+
"Programming Language :: Python :: 3.12",
|
|
17
|
+
"Programming Language :: Python :: 3.13",
|
|
18
|
+
"Programming Language :: Python :: 3.14",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
20
|
+
"Typing :: Typed",
|
|
21
|
+
]
|
|
22
|
+
# Pydantic's maintained HTTPX; 2.15 and later need Python 3.11, so 3.10 resolves to 2.14.
|
|
23
|
+
dependencies = ["httpx2>=2.12,<3"]
|
|
24
|
+
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
# `transport="quic"`: the same calls over HTTP/3. aioquic 1.4 and later need Python 3.11, so 3.10
|
|
27
|
+
# resolves to 1.3.
|
|
28
|
+
quic = ["aioquic>=1.3,<2"]
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Homepage = "https://abstract.inc"
|
|
32
|
+
Documentation = "https://docs.abstract.inc"
|
|
33
|
+
|
|
34
|
+
[build-system]
|
|
35
|
+
requires = ["hatchling"]
|
|
36
|
+
build-backend = "hatchling.build"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.targets.wheel]
|
|
39
|
+
packages = ["src/abstract", "src/atoms"] # atoms: the 0.2 import name, deprecated
|