macbot 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- macbot/__init__.py +3 -0
- macbot/cameras/__init__.py +239 -0
- macbot/cameras/swift/uvc_stream.swift +83 -0
- macbot/cli.py +179 -0
- macbot/collect/__init__.py +0 -0
- macbot/collect/checks.py +36 -0
- macbot/collect/server.py +371 -0
- macbot/config.py +56 -0
- macbot/data/__init__.py +0 -0
- macbot/data/export.py +116 -0
- macbot/data/health.py +67 -0
- macbot/default.yaml +55 -0
- macbot/doctor.py +86 -0
- macbot/i18n.py +10 -0
- macbot/policy/__init__.py +0 -0
- macbot/policy/bench.py +28 -0
- macbot/policy/executors.py +222 -0
- macbot/policy/gripper.py +47 -0
- macbot/policy/models.py +158 -0
- macbot/policy/runner.py +345 -0
- macbot/robot/__init__.py +3 -0
- macbot/robot/fake.py +63 -0
- macbot/robot/gravity.py +108 -0
- macbot/robot/rebot.py +223 -0
- macbot/system.py +97 -0
- macbot/viz/__init__.py +0 -0
- macbot/viz/replay.py +113 -0
- macbot/viz/video.py +83 -0
- macbot/web/__init__.py +85 -0
- macbot/web/static/common.js +139 -0
- macbot/web/static/fonts/JetBrainsMono.ttf +0 -0
- macbot/web/static/fonts/OFL-JetBrainsMono.txt +93 -0
- macbot/web/static/logo-dark.svg +1 -0
- macbot/web/static/logo-mark.svg +1 -0
- macbot/web/static/logo.svg +1 -0
- macbot/web/static/theme.css +200 -0
- macbot/web/templates/_head.html +5 -0
- macbot/web/templates/collect.html +223 -0
- macbot/web/templates/replay.html +132 -0
- macbot/web/templates/run.html +124 -0
- macbot-0.1.0.dist-info/METADATA +183 -0
- macbot-0.1.0.dist-info/RECORD +46 -0
- macbot-0.1.0.dist-info/WHEEL +5 -0
- macbot-0.1.0.dist-info/entry_points.txt +2 -0
- macbot-0.1.0.dist-info/licenses/LICENSE +202 -0
- macbot-0.1.0.dist-info/top_level.txt +1 -0
macbot/doctor.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""``macbot doctor``: check that this Mac is ready, without touching the robot."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import importlib.util
|
|
5
|
+
import os
|
|
6
|
+
import platform
|
|
7
|
+
import shutil
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from .config import Config
|
|
11
|
+
from .system import user_home
|
|
12
|
+
|
|
13
|
+
OK, WARN, BAD = "ok ", "! ", "x "
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def _line(mark, what, detail=""):
|
|
17
|
+
print(f" {mark} {what:<28s} {detail}")
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def run(cfg: Config):
|
|
21
|
+
bad = 0
|
|
22
|
+
print(f"macOS {platform.mac_ver()[0]} on {platform.machine()}, Python {platform.python_version()}")
|
|
23
|
+
print("\npackages")
|
|
24
|
+
for mod, need in [("lerobot", True), ("torch", True), ("cv2", True), ("yaml", True), ("pyorbbecsdk", False),
|
|
25
|
+
("transformers", False)]:
|
|
26
|
+
found = importlib.util.find_spec(mod) is not None
|
|
27
|
+
ver = ""
|
|
28
|
+
if found:
|
|
29
|
+
try:
|
|
30
|
+
ver = getattr(__import__(mod), "__version__", "")
|
|
31
|
+
except Exception as e: # noqa: BLE001
|
|
32
|
+
ver = f"import failed: {e}"
|
|
33
|
+
_line(OK if found else (BAD if need else WARN), mod, ver or ("missing" + ("" if need else " (optional)")))
|
|
34
|
+
bad += need and not found
|
|
35
|
+
|
|
36
|
+
print("\ncompute")
|
|
37
|
+
try:
|
|
38
|
+
import torch
|
|
39
|
+
mps = torch.backends.mps.is_available()
|
|
40
|
+
_line(OK if mps else BAD, "PyTorch MPS", "available" if mps else "not available: policies would run on CPU")
|
|
41
|
+
bad += not mps
|
|
42
|
+
except Exception as e: # noqa: BLE001
|
|
43
|
+
_line(BAD, "PyTorch", str(e))
|
|
44
|
+
bad += 1
|
|
45
|
+
|
|
46
|
+
print("\nrobot")
|
|
47
|
+
lib = Path("/usr/local/lib/libPCBUSB.dylib")
|
|
48
|
+
_line(OK if lib.exists() else BAD, "PCAN-USB driver (PCBUSB)", str(lib) if lib.exists() else "install it, see docs/setup_macos.md")
|
|
49
|
+
bad += not lib.exists()
|
|
50
|
+
from .robot.rebot import find_leader_port
|
|
51
|
+
try:
|
|
52
|
+
_line(OK, "leader arm (USB serial)", find_leader_port())
|
|
53
|
+
except Exception as e: # noqa: BLE001
|
|
54
|
+
_line(WARN, "leader arm (USB serial)", str(e))
|
|
55
|
+
calib = user_home() / ".cache/huggingface/lerobot/calibration/robots"
|
|
56
|
+
found = sorted(f"{p.parent.name}/{p.name}" for p in calib.glob("*/*.json")) if calib.is_dir() else []
|
|
57
|
+
_line(OK if found else WARN, "follower calibration", ", ".join(found) or f"none in {calib}")
|
|
58
|
+
urdf = cfg["robot"].get("urdf")
|
|
59
|
+
_line(OK if urdf else WARN, "URDF for gravity feedforward", urdf or "not set: gravity compensation stays off")
|
|
60
|
+
|
|
61
|
+
print("\ncameras")
|
|
62
|
+
sw = shutil.which("swiftc")
|
|
63
|
+
_line(OK if sw else BAD, "swiftc (Xcode tools)", sw or "run: xcode-select --install")
|
|
64
|
+
bad += not sw
|
|
65
|
+
if sw:
|
|
66
|
+
from .cameras import list_cameras, resolve_uid
|
|
67
|
+
try:
|
|
68
|
+
cams = list_cameras()
|
|
69
|
+
ids = {c["unique_id"] for c in cams}
|
|
70
|
+
for c in cams:
|
|
71
|
+
_line(OK, c["name"][:28], c["unique_id"])
|
|
72
|
+
for name, c in cfg["cameras"].items():
|
|
73
|
+
if c["type"] == "uvc":
|
|
74
|
+
uid = resolve_uid(str(c["unique_id"]))
|
|
75
|
+
_line(OK if uid in ids else WARN, f"config: {name}",
|
|
76
|
+
uid if uid in ids else f"uniqueID {c['unique_id']} is not connected (see macbot cameras)")
|
|
77
|
+
except Exception as e: # noqa: BLE001
|
|
78
|
+
_line(WARN, "camera list", str(e))
|
|
79
|
+
if any(c["type"] == "orbbec" for c in cfg["cameras"].values()):
|
|
80
|
+
root = os.geteuid() == 0
|
|
81
|
+
_line(OK if root else WARN, "Orbbec camera", "running as root" if root else "needs sudo -E (the Orbbec SDK opens USB as root)")
|
|
82
|
+
|
|
83
|
+
print("\nfolders")
|
|
84
|
+
_line(OK, "data_root", str(cfg.root))
|
|
85
|
+
print("\n" + ("all required checks passed" if not bad else f"{bad} required check(s) failed"))
|
|
86
|
+
return 1 if bad else 0
|
macbot/i18n.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Bilingual messages for the web pages. The page shows ``en`` or ``zh`` depending on its language switch."""
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def tr(en: str, zh: str) -> dict:
|
|
5
|
+
return {"en": en, "zh": zh}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def text(m, lang: str = "en") -> str:
|
|
9
|
+
"""Plain text of a message that may be a str or a ``tr`` dict."""
|
|
10
|
+
return m.get(lang) or m.get("en", "") if isinstance(m, dict) else str(m)
|
|
File without changes
|
macbot/policy/bench.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""Time policy inference on this Mac with blank camera images of the configured sizes."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import time
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from ..config import Config
|
|
9
|
+
from ..system import expand
|
|
10
|
+
from .models import load_policy
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def run(cfg: Config, args):
|
|
14
|
+
pol = load_policy(expand(args.policy), args.device, cfg["task"]["instruction"])
|
|
15
|
+
if pol.kind == "pi05" and not args.no_compile:
|
|
16
|
+
pol.enable_compile()
|
|
17
|
+
print("torch.compile on; the first calls compile (~25 s)", flush=True)
|
|
18
|
+
imgs = {c: np.zeros((v["save_size"][1], v["save_size"][0], 3), np.uint8) for c, v in cfg["cameras"].items()}
|
|
19
|
+
for _ in range(3):
|
|
20
|
+
pol(imgs, [0.0] * 7)
|
|
21
|
+
ts = []
|
|
22
|
+
for _ in range(args.n):
|
|
23
|
+
t = time.perf_counter()
|
|
24
|
+
pol(imgs, [0.0] * 7)
|
|
25
|
+
ts.append((time.perf_counter() - t) * 1e3)
|
|
26
|
+
ts = np.array(ts)
|
|
27
|
+
print(f"{pol.kind} on {args.device}, chunk {pol.H}: median {np.median(ts):.0f} ms, "
|
|
28
|
+
f"p90 {np.percentile(ts, 90):.0f} ms, min {ts.min():.0f} ms over {args.n} calls")
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""How action chunks become a stream of joint targets.
|
|
2
|
+
|
|
3
|
+
Every executor exposes the same small interface, so new execution schemes plug in without touching the
|
|
4
|
+
control loop:
|
|
5
|
+
|
|
6
|
+
ex.start() start background threads (if any)
|
|
7
|
+
ex.reset(active) drop all chunks; active=True starts inferring from the current observation
|
|
8
|
+
ex.target(now) -> (a, id) joint target (7,) for wall time ``now``, or (None, id) if nothing yet
|
|
9
|
+
ex.err set to a message if inference failed (the loop pauses)
|
|
10
|
+
|
|
11
|
+
Executors receive ``observe() -> (images, q)`` and ``log(t_chunk, infer_ms, **extra)``; time runs at
|
|
12
|
+
``rate`` policy steps per second (fps x speed). Within a chunk, targets are linearly interpolated.
|
|
13
|
+
|
|
14
|
+
* ``SyncChunk``: infer, execute the first K steps, infer again (the robot waits during inference).
|
|
15
|
+
* ``Ensemble``: temporal ensembling as in ACT. A thread infers continuously; each target is the
|
|
16
|
+
exp(m * age)-weighted mean of all chunks that cover the current time.
|
|
17
|
+
* ``AsyncChunk``: a thread infers while the current chunk keeps executing. The new chunk is aligned in
|
|
18
|
+
time with the old one (step 0 of the new chunk = the step the old chunk was at when inference
|
|
19
|
+
started). With ``guide=True`` (pi0.5 only) the unexecuted rest of the old chunk guides the new one
|
|
20
|
+
(Real-Time Chunking, Black et al. 2025): ``delay`` = expected inference time in steps.
|
|
21
|
+
Override ``accept()`` to decide whether and where to switch to a new chunk.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import threading
|
|
26
|
+
import time
|
|
27
|
+
|
|
28
|
+
import numpy as np
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _interp(ch: np.ndarray, f: float) -> np.ndarray:
|
|
32
|
+
f = min(max(f, 0.0), len(ch) - 1.0)
|
|
33
|
+
i = min(int(f), len(ch) - 2)
|
|
34
|
+
w = f - i
|
|
35
|
+
return ch[i] * (1 - w) + ch[i + 1] * w
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Executor:
|
|
39
|
+
name = "base"
|
|
40
|
+
|
|
41
|
+
def __init__(self, policy, observe, rate: float, log):
|
|
42
|
+
self.pol, self.observe, self.rate, self.log = policy, observe, rate, log
|
|
43
|
+
self.err = None
|
|
44
|
+
|
|
45
|
+
def start(self):
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
def reset(self, active: bool):
|
|
49
|
+
raise NotImplementedError
|
|
50
|
+
|
|
51
|
+
def target(self, now: float):
|
|
52
|
+
raise NotImplementedError
|
|
53
|
+
|
|
54
|
+
def describe(self) -> str:
|
|
55
|
+
return self.name
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class SyncChunk(Executor):
|
|
59
|
+
name = "chunk"
|
|
60
|
+
|
|
61
|
+
def __init__(self, policy, observe, rate, log, k: int):
|
|
62
|
+
super().__init__(policy, observe, rate, log)
|
|
63
|
+
self.k = k
|
|
64
|
+
self.chunk, self.t0, self.id = None, 0.0, -1
|
|
65
|
+
|
|
66
|
+
def reset(self, active: bool):
|
|
67
|
+
self.chunk = None
|
|
68
|
+
|
|
69
|
+
def target(self, now):
|
|
70
|
+
f = (now - self.t0) * self.rate if self.chunk is not None else None
|
|
71
|
+
if self.chunk is None or f >= self.k:
|
|
72
|
+
imgs, q = self.observe()
|
|
73
|
+
t = time.perf_counter()
|
|
74
|
+
try:
|
|
75
|
+
self.chunk = self.pol(imgs, q)
|
|
76
|
+
except Exception as e:
|
|
77
|
+
self.err, self.chunk = repr(e), None
|
|
78
|
+
return None, self.id
|
|
79
|
+
ms = (time.perf_counter() - t) * 1e3
|
|
80
|
+
self.t0, self.id, f = time.time(), self.id + 1, 0.0
|
|
81
|
+
self.log(self.t0, ms)
|
|
82
|
+
return _interp(self.chunk, f), self.id
|
|
83
|
+
|
|
84
|
+
def describe(self):
|
|
85
|
+
return f"chunk (K={self.k}, synchronous)"
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Ensemble(Executor, threading.Thread):
|
|
89
|
+
name = "ensemble"
|
|
90
|
+
start = threading.Thread.start # Executor.start would shadow it in the MRO
|
|
91
|
+
|
|
92
|
+
def __init__(self, policy, observe, rate, log, m: float = 0.01, max_hz: float = 30.0, stop=lambda: False):
|
|
93
|
+
Executor.__init__(self, policy, observe, rate, log)
|
|
94
|
+
threading.Thread.__init__(self, daemon=True)
|
|
95
|
+
self.m, self.min_dt, self.stop = m, 1.0 / max_hz, stop
|
|
96
|
+
self.lk = threading.Lock()
|
|
97
|
+
self.buf, self.active, self.gen, self.n = [], False, 0, 0
|
|
98
|
+
|
|
99
|
+
def reset(self, active: bool):
|
|
100
|
+
with self.lk:
|
|
101
|
+
self.buf, self.active = [], active
|
|
102
|
+
self.gen += 1
|
|
103
|
+
|
|
104
|
+
def target(self, now):
|
|
105
|
+
with self.lk:
|
|
106
|
+
live = [((now - t) * self.rate, ch) for t, ch in self.buf]
|
|
107
|
+
self.buf = [b for b, (f, ch) in zip(self.buf, live) if f < len(ch) - 1]
|
|
108
|
+
n = self.n
|
|
109
|
+
live = [(f, ch) for f, ch in live if 0 <= f < len(ch) - 1]
|
|
110
|
+
if not live:
|
|
111
|
+
return None, n - 1
|
|
112
|
+
fs = np.array([f for f, _ in live])
|
|
113
|
+
w = np.exp(self.m * (fs - fs.max())) # older chunks weigh more, as in ACT
|
|
114
|
+
a = sum(wi * _interp(ch, f) for wi, (f, ch) in zip(w, live)) / w.sum()
|
|
115
|
+
return a, n - 1
|
|
116
|
+
|
|
117
|
+
def run(self):
|
|
118
|
+
while not self.stop():
|
|
119
|
+
with self.lk:
|
|
120
|
+
act, gen = self.active, self.gen
|
|
121
|
+
if not act:
|
|
122
|
+
time.sleep(0.005)
|
|
123
|
+
continue
|
|
124
|
+
t0 = time.perf_counter()
|
|
125
|
+
try:
|
|
126
|
+
imgs, q = self.observe()
|
|
127
|
+
t_obs = time.time()
|
|
128
|
+
chunk = self.pol(imgs, q)
|
|
129
|
+
except Exception as e:
|
|
130
|
+
self.err = repr(e)
|
|
131
|
+
self.reset(False)
|
|
132
|
+
continue
|
|
133
|
+
ms = (time.perf_counter() - t0) * 1e3
|
|
134
|
+
with self.lk:
|
|
135
|
+
if gen != self.gen or not self.active:
|
|
136
|
+
continue
|
|
137
|
+
self.buf.append((t_obs, chunk))
|
|
138
|
+
self.n += 1
|
|
139
|
+
self.log(t_obs, ms)
|
|
140
|
+
time.sleep(max(0.0, self.min_dt - (time.perf_counter() - t0)))
|
|
141
|
+
|
|
142
|
+
def describe(self):
|
|
143
|
+
return f"temporal ensemble (m={self.m})"
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
class AsyncChunk(Executor, threading.Thread):
|
|
147
|
+
name = "async"
|
|
148
|
+
start = threading.Thread.start
|
|
149
|
+
|
|
150
|
+
def __init__(self, policy, observe, rate, log, guide: bool = False, horizon: int = 20, stop=lambda: False):
|
|
151
|
+
Executor.__init__(self, policy, observe, rate, log)
|
|
152
|
+
threading.Thread.__init__(self, daemon=True)
|
|
153
|
+
self.guide, self.horizon, self.stop = guide, horizon, stop
|
|
154
|
+
self.lk = threading.Lock()
|
|
155
|
+
self.cur, self.active, self.gen, self.n = None, False, 0, 0
|
|
156
|
+
self.lat = [] # recent inference times (s)
|
|
157
|
+
|
|
158
|
+
def reset(self, active: bool):
|
|
159
|
+
with self.lk:
|
|
160
|
+
self.cur, self.active = None, active
|
|
161
|
+
self.gen += 1
|
|
162
|
+
|
|
163
|
+
def target(self, now):
|
|
164
|
+
with self.lk:
|
|
165
|
+
cur = self.cur
|
|
166
|
+
if cur is None:
|
|
167
|
+
return None, self.n - 1
|
|
168
|
+
return _interp(cur["act"], (now - cur["T0"]) * self.rate), cur["id"]
|
|
169
|
+
|
|
170
|
+
def accept(self, new_act, new_raw, cur, t_req, t_done, T_aligned):
|
|
171
|
+
"""Return the wall time of step 0 of the new chunk, or None to keep executing ``cur``.
|
|
172
|
+
Default: always switch, time-aligned with the old chunk (first chunk starts now)."""
|
|
173
|
+
return t_done if cur is None else T_aligned
|
|
174
|
+
|
|
175
|
+
def run(self):
|
|
176
|
+
while not self.stop():
|
|
177
|
+
with self.lk:
|
|
178
|
+
act, gen, cur = self.active, self.gen, self.cur
|
|
179
|
+
if not act:
|
|
180
|
+
time.sleep(0.005)
|
|
181
|
+
continue
|
|
182
|
+
t_req = time.time()
|
|
183
|
+
leftover, delay, T_aligned = None, 0, t_req
|
|
184
|
+
if cur is not None:
|
|
185
|
+
i0 = int((t_req - cur["T0"]) * self.rate)
|
|
186
|
+
est = max(self.lat[-5:]) * 1.1 if self.lat else 1.0
|
|
187
|
+
delay = int(np.ceil(est * self.rate)) + 1
|
|
188
|
+
left = cur["raw"][i0:] if cur["raw"] is not None else []
|
|
189
|
+
if len(left) > delay:
|
|
190
|
+
leftover, T_aligned = (left if self.guide else None), cur["T0"] + i0 / self.rate
|
|
191
|
+
else: # old chunk almost done: align to the observation
|
|
192
|
+
delay = 0
|
|
193
|
+
try:
|
|
194
|
+
imgs, q = self.observe()
|
|
195
|
+
if self.pol.kind == "pi05":
|
|
196
|
+
a, raw = self.pol(imgs, q, leftover=leftover, delay=delay, horizon=self.horizon, return_raw=True)
|
|
197
|
+
else:
|
|
198
|
+
a, raw = self.pol(imgs, q), None
|
|
199
|
+
except Exception as e:
|
|
200
|
+
self.err = repr(e)
|
|
201
|
+
self.reset(False)
|
|
202
|
+
continue
|
|
203
|
+
t_done = time.time()
|
|
204
|
+
self.lat = (self.lat + [t_done - t_req])[-20:]
|
|
205
|
+
T0 = self.accept(a, raw, cur, t_req, t_done, T_aligned)
|
|
206
|
+
if T0 is None:
|
|
207
|
+
self.log(None, (t_done - t_req) * 1e3, switched=0)
|
|
208
|
+
continue
|
|
209
|
+
with self.lk:
|
|
210
|
+
if gen != self.gen or not self.active:
|
|
211
|
+
continue
|
|
212
|
+
jump = 0.0 # arm target jump at the switch (deg)
|
|
213
|
+
if self.cur is not None:
|
|
214
|
+
jump = float(np.abs(_interp(a, (t_done - T0) * self.rate)[:6]
|
|
215
|
+
- _interp(self.cur["act"], (t_done - self.cur["T0"]) * self.rate)[:6]).max())
|
|
216
|
+
self.cur = {"act": a, "raw": raw, "T0": T0, "id": self.n}
|
|
217
|
+
self.n += 1
|
|
218
|
+
self.log(T0, (t_done - t_req) * 1e3, switched=1, delay=delay, jump=round(jump, 3),
|
|
219
|
+
overrun=round((t_done - T0) * self.rate - delay, 2))
|
|
220
|
+
|
|
221
|
+
def describe(self):
|
|
222
|
+
return "real-time chunking (guided)" if self.guide else "asynchronous chunks"
|
macbot/policy/gripper.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Binary gripper: the policy outputs a close probability, the robot gets an open or close angle.
|
|
2
|
+
|
|
3
|
+
Datasets exported with ``macbot export --gripper binary`` carry a ``gripper.json`` with the two angles.
|
|
4
|
+
Hysteresis: close as soon as p > 0.6. Once closed, open only after p < 0.4 for ``debounce`` seconds, so a
|
|
5
|
+
short spurious "open" does not drop the object. Keep ``debounce`` shorter than the shortest open phase in
|
|
6
|
+
your demonstrations (for example, a quick release of the last object).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BinaryGripper:
|
|
15
|
+
def __init__(self, cfg: dict, debounce: float):
|
|
16
|
+
self.open_deg, self.close_deg = float(cfg["open_deg"]), float(cfg["close_deg"])
|
|
17
|
+
self.debounce = debounce
|
|
18
|
+
self.closed = False
|
|
19
|
+
self._open_since = None
|
|
20
|
+
|
|
21
|
+
def __call__(self, p: float, t: float) -> float:
|
|
22
|
+
if not self.closed:
|
|
23
|
+
self._open_since = None
|
|
24
|
+
self.closed = p > 0.6
|
|
25
|
+
elif p < 0.4:
|
|
26
|
+
if self._open_since is None:
|
|
27
|
+
self._open_since = t
|
|
28
|
+
if t - self._open_since >= self.debounce:
|
|
29
|
+
self._open_since, self.closed = None, False
|
|
30
|
+
else:
|
|
31
|
+
self._open_since = None
|
|
32
|
+
return self.close_deg if self.closed else self.open_deg
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def find_gripper_config(policy: Path, arg: str) -> dict | None:
|
|
36
|
+
"""``auto``: look for gripper.json in the policy folder and up to three parents. ``raw``: none."""
|
|
37
|
+
if arg == "raw":
|
|
38
|
+
return None
|
|
39
|
+
cands = [Path(arg).expanduser()] if arg != "auto" else [d / "gripper.json" for d in [policy, *list(policy.parents)[:3]]]
|
|
40
|
+
for c in cands:
|
|
41
|
+
if c.is_file():
|
|
42
|
+
cfg = json.loads(c.read_text())
|
|
43
|
+
print(f"binary gripper ({c}): open {cfg['open_deg']} deg, closed {cfg['close_deg']} deg", flush=True)
|
|
44
|
+
return cfg
|
|
45
|
+
if arg != "auto":
|
|
46
|
+
raise FileNotFoundError(arg)
|
|
47
|
+
return None
|
macbot/policy/models.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Policy wrappers: observation (camera images, measured joints) -> action chunk (H, 7) in degrees.
|
|
2
|
+
|
|
3
|
+
Both wrappers run on Apple silicon through PyTorch MPS.
|
|
4
|
+
|
|
5
|
+
pi0.5 notes (all measured on an M-series MacBook):
|
|
6
|
+
* Loading: the model constructor randomly initializes about 4B parameters (~80 s) that the checkpoint
|
|
7
|
+
then overwrites. We turn ``normal_`` and ``uniform_`` into no-ops while constructing.
|
|
8
|
+
* Precision: weights in bfloat16 under MPS autocast. The noise is created as float32 and MPS refuses
|
|
9
|
+
matmuls with mixed dtypes, so autocast is required.
|
|
10
|
+
* Empty camera slots: the base checkpoint has three image slots. With two cameras the third is a masked
|
|
11
|
+
blank image, which still costs a vision pass and 256 prefix tokens. Masked tokens take no part in
|
|
12
|
+
attention, so we drop them; outputs are identical and 10 denoising steps go from ~523 to ~453 ms.
|
|
13
|
+
* ``torch.compile``: each denoising step dispatches ~4300 small kernels and the CPU-side dispatch
|
|
14
|
+
dominates. Compiling ``denoise_step`` and the Gemma prefix forward brings 10 steps from ~412 to
|
|
15
|
+
~282 ms. Compiling the SigLIP vision tower makes it slower, so it is left alone. First call compiles (~25 s).
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import sys
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
import cv2
|
|
24
|
+
import numpy as np
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def prep_image(img_bgr, size, quality: int):
|
|
28
|
+
"""Same path as recording: resize (INTER_AREA) -> JPEG -> decode -> RGB. Returns (RGB uint8, JPEG bytes)."""
|
|
29
|
+
small = cv2.resize(img_bgr, tuple(size), interpolation=cv2.INTER_AREA)
|
|
30
|
+
_, jpg = cv2.imencode(".jpg", small, [cv2.IMWRITE_JPEG_QUALITY, quality])
|
|
31
|
+
rgb = cv2.cvtColor(cv2.imdecode(jpg, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
|
|
32
|
+
return rgb, jpg.reshape(-1)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _to_obs(torch, imgs: dict, q, task: str, rename: dict | None = None):
|
|
36
|
+
obs = {"observation.state": torch.tensor(np.asarray(q, np.float32)).unsqueeze(0)}
|
|
37
|
+
for cam, img in imgs.items():
|
|
38
|
+
key = (rename or {}).get(cam, cam)
|
|
39
|
+
obs[f"observation.images.{key}"] = torch.from_numpy(img).permute(2, 0, 1).float().div(255).unsqueeze(0)
|
|
40
|
+
obs["task"] = [task]
|
|
41
|
+
return obs
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class ACT:
|
|
45
|
+
kind = "act"
|
|
46
|
+
supports_rtc = False
|
|
47
|
+
|
|
48
|
+
def __init__(self, path: Path, device: str, task: str):
|
|
49
|
+
import torch
|
|
50
|
+
from lerobot.policies.act.modeling_act import ACTPolicy
|
|
51
|
+
from lerobot.policies.factory import make_pre_post_processors
|
|
52
|
+
self.torch, self.device, self.task = torch, device, task
|
|
53
|
+
self.pol = ACTPolicy.from_pretrained(path)
|
|
54
|
+
self.pol.config.device = device
|
|
55
|
+
self.pol.to(device).eval()
|
|
56
|
+
self.pre, self.post = make_pre_post_processors(
|
|
57
|
+
self.pol.config, pretrained_path=str(path), preprocessor_overrides={"device_processor": {"device": device}})
|
|
58
|
+
feats = self.pol.config.input_features
|
|
59
|
+
self.shapes = {k.removeprefix("observation.images."): tuple(feats[k].shape)
|
|
60
|
+
for k in feats if k.startswith("observation.images.")} # (3, H, W)
|
|
61
|
+
self.H = self.pol.config.chunk_size
|
|
62
|
+
|
|
63
|
+
def __call__(self, imgs: dict, q, **_) -> np.ndarray:
|
|
64
|
+
torch = self.torch
|
|
65
|
+
with torch.inference_mode():
|
|
66
|
+
a = self.post(self.pol.predict_action_chunk(self.pre(_to_obs(torch, imgs, q, self.task))))
|
|
67
|
+
if self.device == "mps":
|
|
68
|
+
torch.mps.synchronize()
|
|
69
|
+
return a.squeeze(0).float().cpu().numpy()
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class PI05:
|
|
73
|
+
kind = "pi05"
|
|
74
|
+
supports_rtc = True
|
|
75
|
+
RENAME = {"top": "base_0_rgb", "wrist": "left_wrist_0_rgb"}
|
|
76
|
+
|
|
77
|
+
def __init__(self, path: Path, device: str, task: str, steps: int | None = None, rename: dict | None = None):
|
|
78
|
+
import torch
|
|
79
|
+
from lerobot.policies.factory import make_pre_post_processors
|
|
80
|
+
from lerobot.policies.pi05.modeling_pi05 import PI05Policy
|
|
81
|
+
self.torch, self.device, self.task = torch, device, task
|
|
82
|
+
self.rename = rename or self.RENAME
|
|
83
|
+
saved = {k: getattr(torch.Tensor, k) for k in ("normal_", "uniform_")}
|
|
84
|
+
try: # skip the random init that the checkpoint overwrites
|
|
85
|
+
for k in saved:
|
|
86
|
+
setattr(torch.Tensor, k, lambda self, *a, **kw: self)
|
|
87
|
+
self.pol = PI05Policy.from_pretrained(path)
|
|
88
|
+
finally:
|
|
89
|
+
for k, v in saved.items():
|
|
90
|
+
setattr(torch.Tensor, k, v)
|
|
91
|
+
self.pol.config.device = device
|
|
92
|
+
if steps:
|
|
93
|
+
self.pol.config.num_inference_steps = steps
|
|
94
|
+
self.pol.to(device=device, dtype=torch.bfloat16).eval()
|
|
95
|
+
self.pre, self.post = make_pre_post_processors(
|
|
96
|
+
self.pol.config, pretrained_path=str(path), preprocessor_overrides={"device_processor": {"device": device}})
|
|
97
|
+
self.shapes = None # pi0.5 resizes to 224 itself
|
|
98
|
+
self.H = self.pol.config.n_action_steps
|
|
99
|
+
self.rtc = False
|
|
100
|
+
self._skip_empty_cams()
|
|
101
|
+
|
|
102
|
+
def _skip_empty_cams(self):
|
|
103
|
+
model = self.pol.model
|
|
104
|
+
orig = model._embed_images
|
|
105
|
+
|
|
106
|
+
def embed(images, img_masks):
|
|
107
|
+
live = [j for j, m in enumerate(img_masks) if bool(m.any())]
|
|
108
|
+
if not live or len(live) == len(img_masks):
|
|
109
|
+
return orig(images, img_masks)
|
|
110
|
+
es = list(orig([images[j] for j in live], [img_masks[j] for j in live]))
|
|
111
|
+
it = iter(es)
|
|
112
|
+
return [next(it) if j in live else es[0][:, :0] for j in range(len(img_masks))]
|
|
113
|
+
model._embed_images = embed
|
|
114
|
+
|
|
115
|
+
def enable_compile(self):
|
|
116
|
+
torch, m = self.torch, self.pol.model
|
|
117
|
+
m.denoise_step = torch.compile(m.denoise_step, dynamic=False)
|
|
118
|
+
lm = m.paligemma_with_expert.paligemma.model.language_model
|
|
119
|
+
lm.forward = torch.compile(lm.forward, dynamic=False) # pi0.5 calls .forward directly
|
|
120
|
+
|
|
121
|
+
def enable_rtc(self, beta: float):
|
|
122
|
+
"""Real-Time Chunking (Black et al., 2025) in LeRobot's guided mode."""
|
|
123
|
+
from lerobot.policies.rtc.configuration_rtc import RTCConfig
|
|
124
|
+
self.pol.config.rtc_config = RTCConfig(enabled=True, mode="guided", max_guidance_weight=beta,
|
|
125
|
+
execution_horizon=self.H)
|
|
126
|
+
self.pol.init_rtc_processor()
|
|
127
|
+
self.pol.requires_grad_(False) # guidance differentiates w.r.t. x_t only
|
|
128
|
+
self.rtc = True
|
|
129
|
+
|
|
130
|
+
def __call__(self, imgs: dict, q, leftover=None, delay: int = 0, horizon: int | None = None, return_raw=False):
|
|
131
|
+
"""``leftover``: unexecuted rest of the current chunk in model space (for RTC guidance)."""
|
|
132
|
+
torch = self.torch
|
|
133
|
+
obs = _to_obs(torch, imgs, q, self.task, self.rename)
|
|
134
|
+
ac = (torch.autocast(self.device, dtype=torch.bfloat16) if self.device in ("mps", "cuda")
|
|
135
|
+
else torch.autocast("cpu", enabled=False))
|
|
136
|
+
kw = {}
|
|
137
|
+
if leftover is not None and len(leftover) > delay > 0:
|
|
138
|
+
kw = dict(prev_chunk_left_over=torch.from_numpy(np.ascontiguousarray(leftover, np.float32)).unsqueeze(0).to(self.device),
|
|
139
|
+
inference_delay=int(delay), execution_horizon=int(min(len(leftover), horizon or len(leftover))))
|
|
140
|
+
with (torch.no_grad() if self.rtc else torch.inference_mode()), ac: # guidance needs autograd
|
|
141
|
+
self.pol.reset()
|
|
142
|
+
raw = self.pol.predict_action_chunk(self.pre(obs), **kw)
|
|
143
|
+
raw_np = raw.float().cpu().numpy().copy()
|
|
144
|
+
a = self.post(raw)
|
|
145
|
+
if self.device == "mps":
|
|
146
|
+
torch.mps.synchronize()
|
|
147
|
+
a = a.float().cpu().numpy()
|
|
148
|
+
a, raw_np = (a[0] if a.ndim == 3 else a), (raw_np[0] if raw_np.ndim == 3 else raw_np)
|
|
149
|
+
return (a, raw_np) if return_raw else a
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def load_policy(path: Path, device: str, task: str, steps: int | None = None):
|
|
153
|
+
typ = json.loads((Path(path) / "config.json").read_text()).get("type", "act")
|
|
154
|
+
if typ == "pi05":
|
|
155
|
+
return PI05(path, device, task, steps)
|
|
156
|
+
if typ == "act":
|
|
157
|
+
return ACT(path, device, task)
|
|
158
|
+
sys.exit(f"unsupported policy type {typ} (ACT and pi0.5 are supported)")
|