metaquest-teleop 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaquest_teleop-0.1.0/LICENSE +21 -0
- metaquest_teleop-0.1.0/PKG-INFO +280 -0
- metaquest_teleop-0.1.0/README.md +248 -0
- metaquest_teleop-0.1.0/pyproject.toml +50 -0
- metaquest_teleop-0.1.0/setup.cfg +4 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/__init__.py +7 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/__main__.py +5 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/cli.py +67 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/client.html +230 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/dashboard.py +126 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/fake.py +132 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/hands.py +217 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/py.typed +0 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop/server.py +267 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop.egg-info/PKG-INFO +280 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop.egg-info/SOURCES.txt +19 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop.egg-info/dependency_links.txt +1 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop.egg-info/entry_points.txt +3 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop.egg-info/requires.txt +6 -0
- metaquest_teleop-0.1.0/src/metaquest_teleop.egg-info/top_level.txt +1 -0
- metaquest_teleop-0.1.0/tests/test_metaquest_teleop.py +183 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lorenzo Vignoli
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: metaquest-teleop
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Meta Quest hand tracking streamed to Python over USB, for teleoperation: wrist poses, 25 joints and 20 joint angles per hand.
|
|
5
|
+
Author: Gregorio Valenti
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/vigno0405/MetaQuestTeleop
|
|
8
|
+
Project-URL: Issues, https://github.com/vigno0405/MetaQuestTeleop/issues
|
|
9
|
+
Keywords: meta quest,quest 3,webxr,hand tracking,teleoperation,robotics,vr
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Operating System :: OS Independent
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Human Machine Interfaces
|
|
21
|
+
Classifier: Topic :: System :: Hardware
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: aiohttp>=3.8
|
|
27
|
+
Requires-Dist: numpy>=1.21
|
|
28
|
+
Requires-Dist: rich>=12
|
|
29
|
+
Provides-Extra: test
|
|
30
|
+
Requires-Dist: pytest>=7; extra == "test"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# MetaQuestTeleop
|
|
34
|
+
|
|
35
|
+
[](https://github.com/vigno0405/MetaQuestTeleop/actions/workflows/ci.yml)
|
|
36
|
+
|
|
37
|
+
Stream hand tracking from a Meta Quest to Python over a USB cable, for teleoperation: both
|
|
38
|
+
wrist poses, all 25 joints of each hand and 20 joint angles per hand, at the highest frame
|
|
39
|
+
rate the headset offers, with a live dashboard in the terminal.
|
|
40
|
+
|
|
41
|
+
```text
|
|
42
|
+
╭─ Quest hand stream ──────────────────────────────────────────────────────────╮
|
|
43
|
+
│ page http://localhost:8012 open in the Quest browser, press Start │
|
|
44
|
+
│ USB ● USB link ready (2G0YC5ZF8T0912) │
|
|
45
|
+
│ headset ● XR session: immersive-ar, 120 Hz, local-floor │
|
|
46
|
+
│ stream 120 frames/s hands changed in 120/s gap p95 8.7 ms 360 frames │
|
|
47
|
+
│ 15:32:42 headset page connected │
|
|
48
|
+
│ 15:32:44 XR session started: immersive-ar, 120 Hz, local-floor │
|
|
49
|
+
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
50
|
+
╭─ RIGHT ● tracked ───────────────────────────────────────────────────────────╮
|
|
51
|
+
│ wrist x +0.163 y +1.134 z -0.400 m │
|
|
52
|
+
│ yaw +17° pitch +0° roll +0° │
|
|
53
|
+
│ pinch 104 mm │
|
|
54
|
+
│ │
|
|
55
|
+
│ flex abd flex flex │
|
|
56
|
+
│ thumb █ 18° +34° ▋ 12° ▍ 6° │
|
|
57
|
+
│ index ▋ 11° -8° ▋ 12° ▍ 8° │
|
|
58
|
+
│ middle █▋ 27° -9° █▊ 29° █▏ 19° │
|
|
59
|
+
│ ring ██▋ 46° -7° ██▉ 49° █▉ 32° │
|
|
60
|
+
│ pinky ███▋ 62° -4° ████ 67° ██▌ 43° │
|
|
61
|
+
│ thumb: CMC, MCP, IP fingers: MCP, PIP, DIP │
|
|
62
|
+
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The headset runs a small web page (WebXR) that sends every frame it renders to this
|
|
66
|
+
computer. `adb reverse` makes the headset's `localhost` reach the computer through the
|
|
67
|
+
cable, and a `localhost` page is allowed to use WebXR without certificates, so there is
|
|
68
|
+
nothing to install on the headset.
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install metaquest-teleop
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
or the latest version straight from GitHub:
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install git+https://github.com/vigno0405/MetaQuestTeleop
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
This installs the `metaquest_teleop` Python package and two commands, `metaquest-teleop`
|
|
83
|
+
(the stream with its dashboard) and `metaquest-teleop-fake` (a stand-in headset for testing).
|
|
84
|
+
Python 3.10 or newer.
|
|
85
|
+
|
|
86
|
+
## What you need
|
|
87
|
+
|
|
88
|
+
- A Meta Quest 2, 3, 3S or Pro.
|
|
89
|
+
- A USB-C **data** cable (some charging cables carry no data).
|
|
90
|
+
- A computer with `adb` (Linux, macOS or Windows).
|
|
91
|
+
|
|
92
|
+
## One-time setup
|
|
93
|
+
|
|
94
|
+
1. **Developer mode.** In the Meta Horizon phone app: Devices → your headset → Headset
|
|
95
|
+
settings → Developer mode → on. The first time, Meta asks you to join or create a free
|
|
96
|
+
developer organisation.
|
|
97
|
+
2. **adb.** Ubuntu or Debian: `sudo apt install adb`. macOS: `brew install --cask
|
|
98
|
+
android-platform-tools`. Windows: download *SDK Platform-Tools* from
|
|
99
|
+
developer.android.com and add its folder to `PATH`.
|
|
100
|
+
3. **Hand tracking**, on the headset: Settings → Movement tracking → Hand and body
|
|
101
|
+
tracking on, and auto-switch between hands and controllers on.
|
|
102
|
+
|
|
103
|
+
## Run
|
|
104
|
+
|
|
105
|
+
1. Plug the headset into the computer and start the stream:
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
metaquest-teleop
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
2. The first time, put the headset on and accept **Allow USB debugging** (tick *always
|
|
112
|
+
allow*). The dashboard's `USB` line turns green.
|
|
113
|
+
3. In the headset, open the **Browser** and go to **http://localhost:8012**.
|
|
114
|
+
4. Press **Start**, accept the immersive session, put the controllers down and look at your
|
|
115
|
+
hands. The dashboard fills in.
|
|
116
|
+
|
|
117
|
+
Press the Meta button to stop streaming; press Start on the page to stream again. `Ctrl-C`
|
|
118
|
+
stops the command.
|
|
119
|
+
|
|
120
|
+
No headset at hand? In a second terminal, `metaquest-teleop-fake` streams synthetic hands
|
|
121
|
+
to the running command, exactly as the headset would.
|
|
122
|
+
|
|
123
|
+
| Option | |
|
|
124
|
+
|---|---|
|
|
125
|
+
| `--port 8012` | port of the page and the stream |
|
|
126
|
+
| `--udp HOST:PORT` | also send every frame as JSON to a UDP address (see below) |
|
|
127
|
+
| `--plain` | one status line per second instead of the dashboard |
|
|
128
|
+
| `--no-adb` | do not set up the USB link; you run `adb reverse tcp:8012 tcp:8012` yourself |
|
|
129
|
+
| `--version` | print the installed version |
|
|
130
|
+
|
|
131
|
+
`python -m metaquest_teleop` is the same as `metaquest-teleop`.
|
|
132
|
+
|
|
133
|
+
## Use it in your code
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from metaquest_teleop import QuestHands
|
|
137
|
+
|
|
138
|
+
with QuestHands() as quest: # serves the page and keeps the USB link up, in the background
|
|
139
|
+
while True:
|
|
140
|
+
frame = quest.wait() # blocks until the next frame
|
|
141
|
+
if frame.right is not None:
|
|
142
|
+
print(frame.right.wrist.position, frame.right.angles)
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
[`examples/minimal.py`](examples/minimal.py) is a runnable version.
|
|
146
|
+
|
|
147
|
+
**`QuestHands(port=8012, adb=True, udp=None, host="127.0.0.1")`**: `start()` and `stop()`,
|
|
148
|
+
or use it in a `with` block. The server listens on the loopback interface only, so nothing
|
|
149
|
+
else on your network can send it frames; the headset reaches it through `adb reverse`.
|
|
150
|
+
|
|
151
|
+
| | |
|
|
152
|
+
|---|---|
|
|
153
|
+
| `quest.wait(timeout=None)` | the next frame; `None` on timeout |
|
|
154
|
+
| `quest.latest()` | the most recent frame without waiting; `None` before the first |
|
|
155
|
+
| `quest.on_frame(callback)` | `callback(frame)` for every frame, on the receiving thread (keep it fast) |
|
|
156
|
+
| `quest.rates()` | frames per second, interval p95 and age of the last frame |
|
|
157
|
+
| `quest.connected`, `quest.headset` | whether the page is connected, and what it reported (mode, frame rate) |
|
|
158
|
+
|
|
159
|
+
**`Frame`**: `seq`, `time` (headset clock, s), `received` (this computer's `time.time()`),
|
|
160
|
+
`head`, `left`, `right` (a `Hand`, or `None` while that hand is not tracked), `to_dict()`.
|
|
161
|
+
|
|
162
|
+
**`Hand`**: `side`, `wrist` (a `Pose`), `positions` (25, 3), `quaternions` (25, 4),
|
|
163
|
+
`joint(name)`, `angles` (5, 4), `angle_dict` (by name), `local_quaternions` (25, 4, each joint
|
|
164
|
+
relative to its parent), `pinch` (thumb tip to index tip, m), `to_dict()`.
|
|
165
|
+
|
|
166
|
+
**`Pose`**: `position` (3,), `quaternion` (4,), `rotation` (3, 3), `matrix` (4, 4).
|
|
167
|
+
|
|
168
|
+
## Other programs and languages: UDP
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
metaquest-teleop --udp 127.0.0.1:9000
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
sends each frame as one JSON datagram:
|
|
175
|
+
|
|
176
|
+
```json
|
|
177
|
+
{"seq": 1834, "time": 52.31, "received": 1762874211.42,
|
|
178
|
+
"head": {"position": [x, y, z], "quaternion": [x, y, z, w]},
|
|
179
|
+
"right": {"wrist": {"position": [x, y, z], "quaternion": [x, y, z, w]},
|
|
180
|
+
"angles": {"thumb_cmc_flex": 0.31, "...": 0.0, "pinky_dip_flex": 0.62},
|
|
181
|
+
"pinch": 0.041,
|
|
182
|
+
"joints": [[x, y, z, qx, qy, qz, qw], "... 25 rows, in joint order"]},
|
|
183
|
+
"left": null}
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
A listener in Python:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
import json, socket
|
|
190
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
191
|
+
sock.bind(("127.0.0.1", 9000))
|
|
192
|
+
while True:
|
|
193
|
+
frame = json.loads(sock.recv(65536))
|
|
194
|
+
print(frame["right"] and frame["right"]["wrist"]["position"])
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## The data
|
|
198
|
+
|
|
199
|
+
**Units and frame.** Metres, radians, quaternions as (x, y, z, w). Poses are in WebXR's
|
|
200
|
+
`local-floor` space: origin on the floor below where the headset was when you pressed
|
|
201
|
+
Start, **+y up**, **−z the direction the headset faced**, +x to its right. Holding the Meta
|
|
202
|
+
button recentres the view, which moves this origin.
|
|
203
|
+
|
|
204
|
+
**Joints.** 25 per hand, in this order (`metaquest_teleop.JOINTS`): `wrist`; thumb
|
|
205
|
+
`metacarpal`, `phalanx-proximal`, `phalanx-distal`, `tip`; then for index, middle, ring and
|
|
206
|
+
pinky `metacarpal`, `phalanx-proximal`, `phalanx-intermediate`, `phalanx-distal`, `tip`.
|
|
207
|
+
Each joint's own axes, on both hands: **−Z along the bone towards the fingertip, −Y out of
|
|
208
|
+
the palm**. A hand is reported whole or not at all.
|
|
209
|
+
|
|
210
|
+
**Angles.** `hand.angles` is (5, 4), one row per finger from thumb to pinky; names in
|
|
211
|
+
`metaquest_teleop.ANGLE_NAMES`:
|
|
212
|
+
|
|
213
|
+
| | column 0 | column 1 | column 2 | column 3 |
|
|
214
|
+
|---|---|---|---|---|
|
|
215
|
+
| thumb | CMC flexion | CMC abduction | MCP flexion | IP flexion |
|
|
216
|
+
| index … pinky | MCP flexion | MCP abduction | PIP flexion | DIP flexion |
|
|
217
|
+
|
|
218
|
+
Flexion is positive towards the palm; abduction is positive towards the thumb side, on
|
|
219
|
+
both hands. Each angle is the bend of a bone against its parent joint, so an open, flat
|
|
220
|
+
hand reads about 0 everywhere except the thumb's CMC, which is measured against the wrist.
|
|
221
|
+
For a different convention, use `local_quaternions` or the raw joint poses.
|
|
222
|
+
|
|
223
|
+
**Rate.** The page asks the headset for its highest frame rate (72 to 120 Hz depending on
|
|
224
|
+
the model) and sends every frame. Hand tracking itself may update less often than the
|
|
225
|
+
display; the dashboard's *hands changed* count shows how many frames per second carry new
|
|
226
|
+
hand data.
|
|
227
|
+
|
|
228
|
+
## Troubleshooting
|
|
229
|
+
|
|
230
|
+
| The dashboard says | Do this |
|
|
231
|
+
|---|---|
|
|
232
|
+
| `adb not found` | install adb (setup step 2) |
|
|
233
|
+
| `no headset on USB` | data cable, developer mode on, headset awake |
|
|
234
|
+
| `headset not authorised` | put the headset on and accept *Allow USB debugging* |
|
|
235
|
+
| `waiting for the page` | open http://localhost:8012 in the headset's browser while `USB` is green |
|
|
236
|
+
| `page open: press Start` | press Start on the page in the headset |
|
|
237
|
+
| `no frames for … s` | the headset is asleep or the session ended: put it on, press Start |
|
|
238
|
+
|
|
239
|
+
On the page in the headset:
|
|
240
|
+
|
|
241
|
+
- **WebXR is not available**: the page must be opened as `http://localhost:8012` through the
|
|
242
|
+
cable, not by the computer's IP address.
|
|
243
|
+
- **Hand tracking is unavailable**: turn hand tracking on and put the controllers down.
|
|
244
|
+
- **Waiting for the computer**: start `metaquest-teleop`. The page reconnects by itself.
|
|
245
|
+
|
|
246
|
+
The USB link is lost when the cable is unplugged or the headset sleeps; the command
|
|
247
|
+
restores it within two seconds while no page is connected. If port 8012 is taken, use
|
|
248
|
+
`--port`.
|
|
249
|
+
|
|
250
|
+
## Development
|
|
251
|
+
|
|
252
|
+
```bash
|
|
253
|
+
git clone https://github.com/vigno0405/MetaQuestTeleop
|
|
254
|
+
cd MetaQuestTeleop
|
|
255
|
+
python -m pip install -e ".[test]"
|
|
256
|
+
python -m pytest
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
```text
|
|
260
|
+
src/metaquest_teleop/
|
|
261
|
+
server.py QuestHands: serves the page, USB link, receives and forwards frames
|
|
262
|
+
hands.py Frame, Hand, Pose, joint angles, the wire format
|
|
263
|
+
client.html the WebXR page that runs in the headset
|
|
264
|
+
dashboard.py the terminal dashboard
|
|
265
|
+
cli.py the metaquest-teleop command: dashboard, --udp, --plain
|
|
266
|
+
fake.py a stand-in headset with synthetic hands
|
|
267
|
+
examples/minimal.py the smallest use of the API
|
|
268
|
+
tests/ python -m pytest
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
### Releasing
|
|
272
|
+
|
|
273
|
+
1. Bump `__version__` in `src/metaquest_teleop/__init__.py` and commit.
|
|
274
|
+
2. Create a GitHub release with the tag `v<version>` (for example `v0.1.1`).
|
|
275
|
+
3. The `publish` workflow builds the package, checks the tag matches the version, and
|
|
276
|
+
uploads it to PyPI.
|
|
277
|
+
|
|
278
|
+
## License
|
|
279
|
+
|
|
280
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
# MetaQuestTeleop
|
|
2
|
+
|
|
3
|
+
[](https://github.com/vigno0405/MetaQuestTeleop/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
Stream hand tracking from a Meta Quest to Python over a USB cable, for teleoperation: both
|
|
6
|
+
wrist poses, all 25 joints of each hand and 20 joint angles per hand, at the highest frame
|
|
7
|
+
rate the headset offers, with a live dashboard in the terminal.
|
|
8
|
+
|
|
9
|
+
```text
|
|
10
|
+
╭─ Quest hand stream ──────────────────────────────────────────────────────────╮
|
|
11
|
+
│ page http://localhost:8012 open in the Quest browser, press Start │
|
|
12
|
+
│ USB ● USB link ready (2G0YC5ZF8T0912) │
|
|
13
|
+
│ headset ● XR session: immersive-ar, 120 Hz, local-floor │
|
|
14
|
+
│ stream 120 frames/s hands changed in 120/s gap p95 8.7 ms 360 frames │
|
|
15
|
+
│ 15:32:42 headset page connected │
|
|
16
|
+
│ 15:32:44 XR session started: immersive-ar, 120 Hz, local-floor │
|
|
17
|
+
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
18
|
+
╭─ RIGHT ● tracked ───────────────────────────────────────────────────────────╮
|
|
19
|
+
│ wrist x +0.163 y +1.134 z -0.400 m │
|
|
20
|
+
│ yaw +17° pitch +0° roll +0° │
|
|
21
|
+
│ pinch 104 mm │
|
|
22
|
+
│ │
|
|
23
|
+
│ flex abd flex flex │
|
|
24
|
+
│ thumb █ 18° +34° ▋ 12° ▍ 6° │
|
|
25
|
+
│ index ▋ 11° -8° ▋ 12° ▍ 8° │
|
|
26
|
+
│ middle █▋ 27° -9° █▊ 29° █▏ 19° │
|
|
27
|
+
│ ring ██▋ 46° -7° ██▉ 49° █▉ 32° │
|
|
28
|
+
│ pinky ███▋ 62° -4° ████ 67° ██▌ 43° │
|
|
29
|
+
│ thumb: CMC, MCP, IP fingers: MCP, PIP, DIP │
|
|
30
|
+
╰──────────────────────────────────────────────────────────────────────────────╯
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
The headset runs a small web page (WebXR) that sends every frame it renders to this
|
|
34
|
+
computer. `adb reverse` makes the headset's `localhost` reach the computer through the
|
|
35
|
+
cable, and a `localhost` page is allowed to use WebXR without certificates, so there is
|
|
36
|
+
nothing to install on the headset.
|
|
37
|
+
|
|
38
|
+
## Install
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install metaquest-teleop
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
or the latest version straight from GitHub:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install git+https://github.com/vigno0405/MetaQuestTeleop
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
This installs the `metaquest_teleop` Python package and two commands, `metaquest-teleop`
|
|
51
|
+
(the stream with its dashboard) and `metaquest-teleop-fake` (a stand-in headset for testing).
|
|
52
|
+
Python 3.10 or newer.
|
|
53
|
+
|
|
54
|
+
## What you need
|
|
55
|
+
|
|
56
|
+
- A Meta Quest 2, 3, 3S or Pro.
|
|
57
|
+
- A USB-C **data** cable (some charging cables carry no data).
|
|
58
|
+
- A computer with `adb` (Linux, macOS or Windows).
|
|
59
|
+
|
|
60
|
+
## One-time setup
|
|
61
|
+
|
|
62
|
+
1. **Developer mode.** In the Meta Horizon phone app: Devices → your headset → Headset
|
|
63
|
+
settings → Developer mode → on. The first time, Meta asks you to join or create a free
|
|
64
|
+
developer organisation.
|
|
65
|
+
2. **adb.** Ubuntu or Debian: `sudo apt install adb`. macOS: `brew install --cask
|
|
66
|
+
android-platform-tools`. Windows: download *SDK Platform-Tools* from
|
|
67
|
+
developer.android.com and add its folder to `PATH`.
|
|
68
|
+
3. **Hand tracking**, on the headset: Settings → Movement tracking → Hand and body
|
|
69
|
+
tracking on, and auto-switch between hands and controllers on.
|
|
70
|
+
|
|
71
|
+
## Run
|
|
72
|
+
|
|
73
|
+
1. Plug the headset into the computer and start the stream:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
metaquest-teleop
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
2. The first time, put the headset on and accept **Allow USB debugging** (tick *always
|
|
80
|
+
allow*). The dashboard's `USB` line turns green.
|
|
81
|
+
3. In the headset, open the **Browser** and go to **http://localhost:8012**.
|
|
82
|
+
4. Press **Start**, accept the immersive session, put the controllers down and look at your
|
|
83
|
+
hands. The dashboard fills in.
|
|
84
|
+
|
|
85
|
+
Press the Meta button to stop streaming; press Start on the page to stream again. `Ctrl-C`
|
|
86
|
+
stops the command.
|
|
87
|
+
|
|
88
|
+
No headset at hand? In a second terminal, `metaquest-teleop-fake` streams synthetic hands
|
|
89
|
+
to the running command, exactly as the headset would.
|
|
90
|
+
|
|
91
|
+
| Option | |
|
|
92
|
+
|---|---|
|
|
93
|
+
| `--port 8012` | port of the page and the stream |
|
|
94
|
+
| `--udp HOST:PORT` | also send every frame as JSON to a UDP address (see below) |
|
|
95
|
+
| `--plain` | one status line per second instead of the dashboard |
|
|
96
|
+
| `--no-adb` | do not set up the USB link; you run `adb reverse tcp:8012 tcp:8012` yourself |
|
|
97
|
+
| `--version` | print the installed version |
|
|
98
|
+
|
|
99
|
+
`python -m metaquest_teleop` is the same as `metaquest-teleop`.
|
|
100
|
+
|
|
101
|
+
## Use it in your code
|
|
102
|
+
|
|
103
|
+
```python
|
|
104
|
+
from metaquest_teleop import QuestHands
|
|
105
|
+
|
|
106
|
+
with QuestHands() as quest: # serves the page and keeps the USB link up, in the background
|
|
107
|
+
while True:
|
|
108
|
+
frame = quest.wait() # blocks until the next frame
|
|
109
|
+
if frame.right is not None:
|
|
110
|
+
print(frame.right.wrist.position, frame.right.angles)
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
[`examples/minimal.py`](examples/minimal.py) is a runnable version.
|
|
114
|
+
|
|
115
|
+
**`QuestHands(port=8012, adb=True, udp=None, host="127.0.0.1")`**: `start()` and `stop()`,
|
|
116
|
+
or use it in a `with` block. The server listens on the loopback interface only, so nothing
|
|
117
|
+
else on your network can send it frames; the headset reaches it through `adb reverse`.
|
|
118
|
+
|
|
119
|
+
| | |
|
|
120
|
+
|---|---|
|
|
121
|
+
| `quest.wait(timeout=None)` | the next frame; `None` on timeout |
|
|
122
|
+
| `quest.latest()` | the most recent frame without waiting; `None` before the first |
|
|
123
|
+
| `quest.on_frame(callback)` | `callback(frame)` for every frame, on the receiving thread (keep it fast) |
|
|
124
|
+
| `quest.rates()` | frames per second, interval p95 and age of the last frame |
|
|
125
|
+
| `quest.connected`, `quest.headset` | whether the page is connected, and what it reported (mode, frame rate) |
|
|
126
|
+
|
|
127
|
+
**`Frame`**: `seq`, `time` (headset clock, s), `received` (this computer's `time.time()`),
|
|
128
|
+
`head`, `left`, `right` (a `Hand`, or `None` while that hand is not tracked), `to_dict()`.
|
|
129
|
+
|
|
130
|
+
**`Hand`**: `side`, `wrist` (a `Pose`), `positions` (25, 3), `quaternions` (25, 4),
|
|
131
|
+
`joint(name)`, `angles` (5, 4), `angle_dict` (by name), `local_quaternions` (25, 4, each joint
|
|
132
|
+
relative to its parent), `pinch` (thumb tip to index tip, m), `to_dict()`.
|
|
133
|
+
|
|
134
|
+
**`Pose`**: `position` (3,), `quaternion` (4,), `rotation` (3, 3), `matrix` (4, 4).
|
|
135
|
+
|
|
136
|
+
## Other programs and languages: UDP
|
|
137
|
+
|
|
138
|
+
```bash
|
|
139
|
+
metaquest-teleop --udp 127.0.0.1:9000
|
|
140
|
+
```
|
|
141
|
+
|
|
142
|
+
sends each frame as one JSON datagram:
|
|
143
|
+
|
|
144
|
+
```json
|
|
145
|
+
{"seq": 1834, "time": 52.31, "received": 1762874211.42,
|
|
146
|
+
"head": {"position": [x, y, z], "quaternion": [x, y, z, w]},
|
|
147
|
+
"right": {"wrist": {"position": [x, y, z], "quaternion": [x, y, z, w]},
|
|
148
|
+
"angles": {"thumb_cmc_flex": 0.31, "...": 0.0, "pinky_dip_flex": 0.62},
|
|
149
|
+
"pinch": 0.041,
|
|
150
|
+
"joints": [[x, y, z, qx, qy, qz, qw], "... 25 rows, in joint order"]},
|
|
151
|
+
"left": null}
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
A listener in Python:
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
import json, socket
|
|
158
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
159
|
+
sock.bind(("127.0.0.1", 9000))
|
|
160
|
+
while True:
|
|
161
|
+
frame = json.loads(sock.recv(65536))
|
|
162
|
+
print(frame["right"] and frame["right"]["wrist"]["position"])
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
## The data
|
|
166
|
+
|
|
167
|
+
**Units and frame.** Metres, radians, quaternions as (x, y, z, w). Poses are in WebXR's
|
|
168
|
+
`local-floor` space: origin on the floor below where the headset was when you pressed
|
|
169
|
+
Start, **+y up**, **−z the direction the headset faced**, +x to its right. Holding the Meta
|
|
170
|
+
button recentres the view, which moves this origin.
|
|
171
|
+
|
|
172
|
+
**Joints.** 25 per hand, in this order (`metaquest_teleop.JOINTS`): `wrist`; thumb
|
|
173
|
+
`metacarpal`, `phalanx-proximal`, `phalanx-distal`, `tip`; then for index, middle, ring and
|
|
174
|
+
pinky `metacarpal`, `phalanx-proximal`, `phalanx-intermediate`, `phalanx-distal`, `tip`.
|
|
175
|
+
Each joint's own axes, on both hands: **−Z along the bone towards the fingertip, −Y out of
|
|
176
|
+
the palm**. A hand is reported whole or not at all.
|
|
177
|
+
|
|
178
|
+
**Angles.** `hand.angles` is (5, 4), one row per finger from thumb to pinky; names in
|
|
179
|
+
`metaquest_teleop.ANGLE_NAMES`:
|
|
180
|
+
|
|
181
|
+
| | column 0 | column 1 | column 2 | column 3 |
|
|
182
|
+
|---|---|---|---|---|
|
|
183
|
+
| thumb | CMC flexion | CMC abduction | MCP flexion | IP flexion |
|
|
184
|
+
| index … pinky | MCP flexion | MCP abduction | PIP flexion | DIP flexion |
|
|
185
|
+
|
|
186
|
+
Flexion is positive towards the palm; abduction is positive towards the thumb side, on
|
|
187
|
+
both hands. Each angle is the bend of a bone against its parent joint, so an open, flat
|
|
188
|
+
hand reads about 0 everywhere except the thumb's CMC, which is measured against the wrist.
|
|
189
|
+
For a different convention, use `local_quaternions` or the raw joint poses.
|
|
190
|
+
|
|
191
|
+
**Rate.** The page asks the headset for its highest frame rate (72 to 120 Hz depending on
|
|
192
|
+
the model) and sends every frame. Hand tracking itself may update less often than the
|
|
193
|
+
display; the dashboard's *hands changed* count shows how many frames per second carry new
|
|
194
|
+
hand data.
|
|
195
|
+
|
|
196
|
+
## Troubleshooting
|
|
197
|
+
|
|
198
|
+
| The dashboard says | Do this |
|
|
199
|
+
|---|---|
|
|
200
|
+
| `adb not found` | install adb (setup step 2) |
|
|
201
|
+
| `no headset on USB` | data cable, developer mode on, headset awake |
|
|
202
|
+
| `headset not authorised` | put the headset on and accept *Allow USB debugging* |
|
|
203
|
+
| `waiting for the page` | open http://localhost:8012 in the headset's browser while `USB` is green |
|
|
204
|
+
| `page open: press Start` | press Start on the page in the headset |
|
|
205
|
+
| `no frames for … s` | the headset is asleep or the session ended: put it on, press Start |
|
|
206
|
+
|
|
207
|
+
On the page in the headset:
|
|
208
|
+
|
|
209
|
+
- **WebXR is not available**: the page must be opened as `http://localhost:8012` through the
|
|
210
|
+
cable, not by the computer's IP address.
|
|
211
|
+
- **Hand tracking is unavailable**: turn hand tracking on and put the controllers down.
|
|
212
|
+
- **Waiting for the computer**: start `metaquest-teleop`. The page reconnects by itself.
|
|
213
|
+
|
|
214
|
+
The USB link is lost when the cable is unplugged or the headset sleeps; the command
|
|
215
|
+
restores it within two seconds while no page is connected. If port 8012 is taken, use
|
|
216
|
+
`--port`.
|
|
217
|
+
|
|
218
|
+
## Development
|
|
219
|
+
|
|
220
|
+
```bash
|
|
221
|
+
git clone https://github.com/vigno0405/MetaQuestTeleop
|
|
222
|
+
cd MetaQuestTeleop
|
|
223
|
+
python -m pip install -e ".[test]"
|
|
224
|
+
python -m pytest
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
```text
|
|
228
|
+
src/metaquest_teleop/
|
|
229
|
+
server.py QuestHands: serves the page, USB link, receives and forwards frames
|
|
230
|
+
hands.py Frame, Hand, Pose, joint angles, the wire format
|
|
231
|
+
client.html the WebXR page that runs in the headset
|
|
232
|
+
dashboard.py the terminal dashboard
|
|
233
|
+
cli.py the metaquest-teleop command: dashboard, --udp, --plain
|
|
234
|
+
fake.py a stand-in headset with synthetic hands
|
|
235
|
+
examples/minimal.py the smallest use of the API
|
|
236
|
+
tests/ python -m pytest
|
|
237
|
+
```
|
|
238
|
+
|
|
239
|
+
### Releasing
|
|
240
|
+
|
|
241
|
+
1. Bump `__version__` in `src/metaquest_teleop/__init__.py` and commit.
|
|
242
|
+
2. Create a GitHub release with the tag `v<version>` (for example `v0.1.1`).
|
|
243
|
+
3. The `publish` workflow builds the package, checks the tag matches the version, and
|
|
244
|
+
uploads it to PyPI.
|
|
245
|
+
|
|
246
|
+
## License
|
|
247
|
+
|
|
248
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "metaquest-teleop"
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Meta Quest hand tracking streamed to Python over USB, for teleoperation: wrist poses, 25 joints and 20 joint angles per hand."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Gregorio Valenti" }]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
dependencies = ["aiohttp>=3.8", "numpy>=1.21", "rich>=12"]
|
|
15
|
+
keywords = ["meta quest", "quest 3", "webxr", "hand tracking", "teleoperation", "robotics", "vr"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 3 - Alpha",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Intended Audience :: Science/Research",
|
|
20
|
+
"Operating System :: OS Independent",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
23
|
+
"Programming Language :: Python :: 3.10",
|
|
24
|
+
"Programming Language :: Python :: 3.11",
|
|
25
|
+
"Programming Language :: Python :: 3.12",
|
|
26
|
+
"Programming Language :: Python :: 3.13",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Human Machine Interfaces",
|
|
28
|
+
"Topic :: System :: Hardware",
|
|
29
|
+
"Typing :: Typed",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.optional-dependencies]
|
|
33
|
+
test = ["pytest>=7"]
|
|
34
|
+
|
|
35
|
+
[project.scripts]
|
|
36
|
+
metaquest-teleop = "metaquest_teleop.cli:main"
|
|
37
|
+
metaquest-teleop-fake = "metaquest_teleop.fake:main"
|
|
38
|
+
|
|
39
|
+
[project.urls]
|
|
40
|
+
Homepage = "https://github.com/vigno0405/MetaQuestTeleop"
|
|
41
|
+
Issues = "https://github.com/vigno0405/MetaQuestTeleop/issues"
|
|
42
|
+
|
|
43
|
+
[tool.setuptools.dynamic]
|
|
44
|
+
version = { attr = "metaquest_teleop.__version__" }
|
|
45
|
+
|
|
46
|
+
[tool.setuptools.package-data]
|
|
47
|
+
metaquest_teleop = ["client.html", "py.typed"]
|
|
48
|
+
|
|
49
|
+
[tool.pytest.ini_options]
|
|
50
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Meta Quest hand tracking, streamed to Python over USB, for teleoperation."""
|
|
2
|
+
|
|
3
|
+
from .hands import ANGLE_NAMES, FINGERS, JOINTS, Frame, Hand, Pose
|
|
4
|
+
from .server import DEFAULT_PORT, QuestHands
|
|
5
|
+
|
|
6
|
+
__all__ = ["ANGLE_NAMES", "DEFAULT_PORT", "FINGERS", "JOINTS", "Frame", "Hand", "Pose", "QuestHands"]
|
|
7
|
+
__version__ = "0.1.0"
|