macbot 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. macbot/__init__.py +3 -0
  2. macbot/cameras/__init__.py +239 -0
  3. macbot/cameras/swift/uvc_stream.swift +83 -0
  4. macbot/cli.py +179 -0
  5. macbot/collect/__init__.py +0 -0
  6. macbot/collect/checks.py +36 -0
  7. macbot/collect/server.py +371 -0
  8. macbot/config.py +56 -0
  9. macbot/data/__init__.py +0 -0
  10. macbot/data/export.py +116 -0
  11. macbot/data/health.py +67 -0
  12. macbot/default.yaml +55 -0
  13. macbot/doctor.py +86 -0
  14. macbot/i18n.py +10 -0
  15. macbot/policy/__init__.py +0 -0
  16. macbot/policy/bench.py +28 -0
  17. macbot/policy/executors.py +222 -0
  18. macbot/policy/gripper.py +47 -0
  19. macbot/policy/models.py +158 -0
  20. macbot/policy/runner.py +345 -0
  21. macbot/robot/__init__.py +3 -0
  22. macbot/robot/fake.py +63 -0
  23. macbot/robot/gravity.py +108 -0
  24. macbot/robot/rebot.py +223 -0
  25. macbot/system.py +97 -0
  26. macbot/viz/__init__.py +0 -0
  27. macbot/viz/replay.py +113 -0
  28. macbot/viz/video.py +83 -0
  29. macbot/web/__init__.py +85 -0
  30. macbot/web/static/common.js +139 -0
  31. macbot/web/static/fonts/JetBrainsMono.ttf +0 -0
  32. macbot/web/static/fonts/OFL-JetBrainsMono.txt +93 -0
  33. macbot/web/static/logo-dark.svg +1 -0
  34. macbot/web/static/logo-mark.svg +1 -0
  35. macbot/web/static/logo.svg +1 -0
  36. macbot/web/static/theme.css +200 -0
  37. macbot/web/templates/_head.html +5 -0
  38. macbot/web/templates/collect.html +223 -0
  39. macbot/web/templates/replay.html +132 -0
  40. macbot/web/templates/run.html +124 -0
  41. macbot-0.1.0.dist-info/METADATA +183 -0
  42. macbot-0.1.0.dist-info/RECORD +46 -0
  43. macbot-0.1.0.dist-info/WHEEL +5 -0
  44. macbot-0.1.0.dist-info/entry_points.txt +2 -0
  45. macbot-0.1.0.dist-info/licenses/LICENSE +202 -0
  46. macbot-0.1.0.dist-info/top_level.txt +1 -0
macbot/doctor.py ADDED
@@ -0,0 +1,86 @@
1
+ """``macbot doctor``: check that this Mac is ready, without touching the robot."""
2
+ from __future__ import annotations
3
+
4
+ import importlib.util
5
+ import os
6
+ import platform
7
+ import shutil
8
+ from pathlib import Path
9
+
10
+ from .config import Config
11
+ from .system import user_home
12
+
13
+ OK, WARN, BAD = "ok ", "! ", "x "
14
+
15
+
16
+ def _line(mark, what, detail=""):
17
+ print(f" {mark} {what:<28s} {detail}")
18
+
19
+
20
+ def run(cfg: Config):
21
+ bad = 0
22
+ print(f"macOS {platform.mac_ver()[0]} on {platform.machine()}, Python {platform.python_version()}")
23
+ print("\npackages")
24
+ for mod, need in [("lerobot", True), ("torch", True), ("cv2", True), ("yaml", True), ("pyorbbecsdk", False),
25
+ ("transformers", False)]:
26
+ found = importlib.util.find_spec(mod) is not None
27
+ ver = ""
28
+ if found:
29
+ try:
30
+ ver = getattr(__import__(mod), "__version__", "")
31
+ except Exception as e: # noqa: BLE001
32
+ ver = f"import failed: {e}"
33
+ _line(OK if found else (BAD if need else WARN), mod, ver or ("missing" + ("" if need else " (optional)")))
34
+ bad += need and not found
35
+
36
+ print("\ncompute")
37
+ try:
38
+ import torch
39
+ mps = torch.backends.mps.is_available()
40
+ _line(OK if mps else BAD, "PyTorch MPS", "available" if mps else "not available: policies would run on CPU")
41
+ bad += not mps
42
+ except Exception as e: # noqa: BLE001
43
+ _line(BAD, "PyTorch", str(e))
44
+ bad += 1
45
+
46
+ print("\nrobot")
47
+ lib = Path("/usr/local/lib/libPCBUSB.dylib")
48
+ _line(OK if lib.exists() else BAD, "PCAN-USB driver (PCBUSB)", str(lib) if lib.exists() else "install it, see docs/setup_macos.md")
49
+ bad += not lib.exists()
50
+ from .robot.rebot import find_leader_port
51
+ try:
52
+ _line(OK, "leader arm (USB serial)", find_leader_port())
53
+ except Exception as e: # noqa: BLE001
54
+ _line(WARN, "leader arm (USB serial)", str(e))
55
+ calib = user_home() / ".cache/huggingface/lerobot/calibration/robots"
56
+ found = sorted(f"{p.parent.name}/{p.name}" for p in calib.glob("*/*.json")) if calib.is_dir() else []
57
+ _line(OK if found else WARN, "follower calibration", ", ".join(found) or f"none in {calib}")
58
+ urdf = cfg["robot"].get("urdf")
59
+ _line(OK if urdf else WARN, "URDF for gravity feedforward", urdf or "not set: gravity compensation stays off")
60
+
61
+ print("\ncameras")
62
+ sw = shutil.which("swiftc")
63
+ _line(OK if sw else BAD, "swiftc (Xcode tools)", sw or "run: xcode-select --install")
64
+ bad += not sw
65
+ if sw:
66
+ from .cameras import list_cameras, resolve_uid
67
+ try:
68
+ cams = list_cameras()
69
+ ids = {c["unique_id"] for c in cams}
70
+ for c in cams:
71
+ _line(OK, c["name"][:28], c["unique_id"])
72
+ for name, c in cfg["cameras"].items():
73
+ if c["type"] == "uvc":
74
+ uid = resolve_uid(str(c["unique_id"]))
75
+ _line(OK if uid in ids else WARN, f"config: {name}",
76
+ uid if uid in ids else f"uniqueID {c['unique_id']} is not connected (see macbot cameras)")
77
+ except Exception as e: # noqa: BLE001
78
+ _line(WARN, "camera list", str(e))
79
+ if any(c["type"] == "orbbec" for c in cfg["cameras"].values()):
80
+ root = os.geteuid() == 0
81
+ _line(OK if root else WARN, "Orbbec camera", "running as root" if root else "needs sudo -E (the Orbbec SDK opens USB as root)")
82
+
83
+ print("\nfolders")
84
+ _line(OK, "data_root", str(cfg.root))
85
+ print("\n" + ("all required checks passed" if not bad else f"{bad} required check(s) failed"))
86
+ return 1 if bad else 0
macbot/i18n.py ADDED
@@ -0,0 +1,10 @@
1
+ """Bilingual messages for the web pages. The page shows ``en`` or ``zh`` depending on its language switch."""
2
+
3
+
4
+ def tr(en: str, zh: str) -> dict:
5
+ return {"en": en, "zh": zh}
6
+
7
+
8
+ def text(m, lang: str = "en") -> str:
9
+ """Plain text of a message that may be a str or a ``tr`` dict."""
10
+ return m.get(lang) or m.get("en", "") if isinstance(m, dict) else str(m)
File without changes
macbot/policy/bench.py ADDED
@@ -0,0 +1,28 @@
1
+ """Time policy inference on this Mac with blank camera images of the configured sizes."""
2
+ from __future__ import annotations
3
+
4
+ import time
5
+
6
+ import numpy as np
7
+
8
+ from ..config import Config
9
+ from ..system import expand
10
+ from .models import load_policy
11
+
12
+
13
+ def run(cfg: Config, args):
14
+ pol = load_policy(expand(args.policy), args.device, cfg["task"]["instruction"])
15
+ if pol.kind == "pi05" and not args.no_compile:
16
+ pol.enable_compile()
17
+ print("torch.compile on; the first calls compile (~25 s)", flush=True)
18
+ imgs = {c: np.zeros((v["save_size"][1], v["save_size"][0], 3), np.uint8) for c, v in cfg["cameras"].items()}
19
+ for _ in range(3):
20
+ pol(imgs, [0.0] * 7)
21
+ ts = []
22
+ for _ in range(args.n):
23
+ t = time.perf_counter()
24
+ pol(imgs, [0.0] * 7)
25
+ ts.append((time.perf_counter() - t) * 1e3)
26
+ ts = np.array(ts)
27
+ print(f"{pol.kind} on {args.device}, chunk {pol.H}: median {np.median(ts):.0f} ms, "
28
+ f"p90 {np.percentile(ts, 90):.0f} ms, min {ts.min():.0f} ms over {args.n} calls")
@@ -0,0 +1,222 @@
1
+ """How action chunks become a stream of joint targets.
2
+
3
+ Every executor exposes the same small interface, so new execution schemes plug in without touching the
4
+ control loop:
5
+
6
+ ex.start() start background threads (if any)
7
+ ex.reset(active) drop all chunks; active=True starts inferring from the current observation
8
+ ex.target(now) -> (a, id) joint target (7,) for wall time ``now``, or (None, id) if nothing yet
9
+ ex.err set to a message if inference failed (the loop pauses)
10
+
11
+ Executors receive ``observe() -> (images, q)`` and ``log(t_chunk, infer_ms, **extra)``; time runs at
12
+ ``rate`` policy steps per second (fps x speed). Within a chunk, targets are linearly interpolated.
13
+
14
+ * ``SyncChunk``: infer, execute the first K steps, infer again (the robot waits during inference).
15
+ * ``Ensemble``: temporal ensembling as in ACT. A thread infers continuously; each target is the
16
+ exp(m * age)-weighted mean of all chunks that cover the current time.
17
+ * ``AsyncChunk``: a thread infers while the current chunk keeps executing. The new chunk is aligned in
18
+ time with the old one (step 0 of the new chunk = the step the old chunk was at when inference
19
+ started). With ``guide=True`` (pi0.5 only) the unexecuted rest of the old chunk guides the new one
20
+ (Real-Time Chunking, Black et al. 2025): ``delay`` = expected inference time in steps.
21
+ Override ``accept()`` to decide whether and where to switch to a new chunk.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import threading
26
+ import time
27
+
28
+ import numpy as np
29
+
30
+
31
+ def _interp(ch: np.ndarray, f: float) -> np.ndarray:
32
+ f = min(max(f, 0.0), len(ch) - 1.0)
33
+ i = min(int(f), len(ch) - 2)
34
+ w = f - i
35
+ return ch[i] * (1 - w) + ch[i + 1] * w
36
+
37
+
38
+ class Executor:
39
+ name = "base"
40
+
41
+ def __init__(self, policy, observe, rate: float, log):
42
+ self.pol, self.observe, self.rate, self.log = policy, observe, rate, log
43
+ self.err = None
44
+
45
+ def start(self):
46
+ pass
47
+
48
+ def reset(self, active: bool):
49
+ raise NotImplementedError
50
+
51
+ def target(self, now: float):
52
+ raise NotImplementedError
53
+
54
+ def describe(self) -> str:
55
+ return self.name
56
+
57
+
58
+ class SyncChunk(Executor):
59
+ name = "chunk"
60
+
61
+ def __init__(self, policy, observe, rate, log, k: int):
62
+ super().__init__(policy, observe, rate, log)
63
+ self.k = k
64
+ self.chunk, self.t0, self.id = None, 0.0, -1
65
+
66
+ def reset(self, active: bool):
67
+ self.chunk = None
68
+
69
+ def target(self, now):
70
+ f = (now - self.t0) * self.rate if self.chunk is not None else None
71
+ if self.chunk is None or f >= self.k:
72
+ imgs, q = self.observe()
73
+ t = time.perf_counter()
74
+ try:
75
+ self.chunk = self.pol(imgs, q)
76
+ except Exception as e:
77
+ self.err, self.chunk = repr(e), None
78
+ return None, self.id
79
+ ms = (time.perf_counter() - t) * 1e3
80
+ self.t0, self.id, f = time.time(), self.id + 1, 0.0
81
+ self.log(self.t0, ms)
82
+ return _interp(self.chunk, f), self.id
83
+
84
+ def describe(self):
85
+ return f"chunk (K={self.k}, synchronous)"
86
+
87
+
88
+ class Ensemble(Executor, threading.Thread):
89
+ name = "ensemble"
90
+ start = threading.Thread.start # Executor.start would shadow it in the MRO
91
+
92
+ def __init__(self, policy, observe, rate, log, m: float = 0.01, max_hz: float = 30.0, stop=lambda: False):
93
+ Executor.__init__(self, policy, observe, rate, log)
94
+ threading.Thread.__init__(self, daemon=True)
95
+ self.m, self.min_dt, self.stop = m, 1.0 / max_hz, stop
96
+ self.lk = threading.Lock()
97
+ self.buf, self.active, self.gen, self.n = [], False, 0, 0
98
+
99
+ def reset(self, active: bool):
100
+ with self.lk:
101
+ self.buf, self.active = [], active
102
+ self.gen += 1
103
+
104
+ def target(self, now):
105
+ with self.lk:
106
+ live = [((now - t) * self.rate, ch) for t, ch in self.buf]
107
+ self.buf = [b for b, (f, ch) in zip(self.buf, live) if f < len(ch) - 1]
108
+ n = self.n
109
+ live = [(f, ch) for f, ch in live if 0 <= f < len(ch) - 1]
110
+ if not live:
111
+ return None, n - 1
112
+ fs = np.array([f for f, _ in live])
113
+ w = np.exp(self.m * (fs - fs.max())) # older chunks weigh more, as in ACT
114
+ a = sum(wi * _interp(ch, f) for wi, (f, ch) in zip(w, live)) / w.sum()
115
+ return a, n - 1
116
+
117
+ def run(self):
118
+ while not self.stop():
119
+ with self.lk:
120
+ act, gen = self.active, self.gen
121
+ if not act:
122
+ time.sleep(0.005)
123
+ continue
124
+ t0 = time.perf_counter()
125
+ try:
126
+ imgs, q = self.observe()
127
+ t_obs = time.time()
128
+ chunk = self.pol(imgs, q)
129
+ except Exception as e:
130
+ self.err = repr(e)
131
+ self.reset(False)
132
+ continue
133
+ ms = (time.perf_counter() - t0) * 1e3
134
+ with self.lk:
135
+ if gen != self.gen or not self.active:
136
+ continue
137
+ self.buf.append((t_obs, chunk))
138
+ self.n += 1
139
+ self.log(t_obs, ms)
140
+ time.sleep(max(0.0, self.min_dt - (time.perf_counter() - t0)))
141
+
142
+ def describe(self):
143
+ return f"temporal ensemble (m={self.m})"
144
+
145
+
146
+ class AsyncChunk(Executor, threading.Thread):
147
+ name = "async"
148
+ start = threading.Thread.start
149
+
150
+ def __init__(self, policy, observe, rate, log, guide: bool = False, horizon: int = 20, stop=lambda: False):
151
+ Executor.__init__(self, policy, observe, rate, log)
152
+ threading.Thread.__init__(self, daemon=True)
153
+ self.guide, self.horizon, self.stop = guide, horizon, stop
154
+ self.lk = threading.Lock()
155
+ self.cur, self.active, self.gen, self.n = None, False, 0, 0
156
+ self.lat = [] # recent inference times (s)
157
+
158
+ def reset(self, active: bool):
159
+ with self.lk:
160
+ self.cur, self.active = None, active
161
+ self.gen += 1
162
+
163
+ def target(self, now):
164
+ with self.lk:
165
+ cur = self.cur
166
+ if cur is None:
167
+ return None, self.n - 1
168
+ return _interp(cur["act"], (now - cur["T0"]) * self.rate), cur["id"]
169
+
170
+ def accept(self, new_act, new_raw, cur, t_req, t_done, T_aligned):
171
+ """Return the wall time of step 0 of the new chunk, or None to keep executing ``cur``.
172
+ Default: always switch, time-aligned with the old chunk (first chunk starts now)."""
173
+ return t_done if cur is None else T_aligned
174
+
175
+ def run(self):
176
+ while not self.stop():
177
+ with self.lk:
178
+ act, gen, cur = self.active, self.gen, self.cur
179
+ if not act:
180
+ time.sleep(0.005)
181
+ continue
182
+ t_req = time.time()
183
+ leftover, delay, T_aligned = None, 0, t_req
184
+ if cur is not None:
185
+ i0 = int((t_req - cur["T0"]) * self.rate)
186
+ est = max(self.lat[-5:]) * 1.1 if self.lat else 1.0
187
+ delay = int(np.ceil(est * self.rate)) + 1
188
+ left = cur["raw"][i0:] if cur["raw"] is not None else []
189
+ if len(left) > delay:
190
+ leftover, T_aligned = (left if self.guide else None), cur["T0"] + i0 / self.rate
191
+ else: # old chunk almost done: align to the observation
192
+ delay = 0
193
+ try:
194
+ imgs, q = self.observe()
195
+ if self.pol.kind == "pi05":
196
+ a, raw = self.pol(imgs, q, leftover=leftover, delay=delay, horizon=self.horizon, return_raw=True)
197
+ else:
198
+ a, raw = self.pol(imgs, q), None
199
+ except Exception as e:
200
+ self.err = repr(e)
201
+ self.reset(False)
202
+ continue
203
+ t_done = time.time()
204
+ self.lat = (self.lat + [t_done - t_req])[-20:]
205
+ T0 = self.accept(a, raw, cur, t_req, t_done, T_aligned)
206
+ if T0 is None:
207
+ self.log(None, (t_done - t_req) * 1e3, switched=0)
208
+ continue
209
+ with self.lk:
210
+ if gen != self.gen or not self.active:
211
+ continue
212
+ jump = 0.0 # arm target jump at the switch (deg)
213
+ if self.cur is not None:
214
+ jump = float(np.abs(_interp(a, (t_done - T0) * self.rate)[:6]
215
+ - _interp(self.cur["act"], (t_done - self.cur["T0"]) * self.rate)[:6]).max())
216
+ self.cur = {"act": a, "raw": raw, "T0": T0, "id": self.n}
217
+ self.n += 1
218
+ self.log(T0, (t_done - t_req) * 1e3, switched=1, delay=delay, jump=round(jump, 3),
219
+ overrun=round((t_done - T0) * self.rate - delay, 2))
220
+
221
+ def describe(self):
222
+ return "real-time chunking (guided)" if self.guide else "asynchronous chunks"
@@ -0,0 +1,47 @@
1
+ """Binary gripper: the policy outputs a close probability, the robot gets an open or close angle.
2
+
3
+ Datasets exported with ``macbot export --gripper binary`` carry a ``gripper.json`` with the two angles.
4
+ Hysteresis: close as soon as p > 0.6. Once closed, open only after p < 0.4 for ``debounce`` seconds, so a
5
+ short spurious "open" does not drop the object. Keep ``debounce`` shorter than the shortest open phase in
6
+ your demonstrations (for example, a quick release of the last object).
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import json
11
+ from pathlib import Path
12
+
13
+
14
+ class BinaryGripper:
15
+ def __init__(self, cfg: dict, debounce: float):
16
+ self.open_deg, self.close_deg = float(cfg["open_deg"]), float(cfg["close_deg"])
17
+ self.debounce = debounce
18
+ self.closed = False
19
+ self._open_since = None
20
+
21
+ def __call__(self, p: float, t: float) -> float:
22
+ if not self.closed:
23
+ self._open_since = None
24
+ self.closed = p > 0.6
25
+ elif p < 0.4:
26
+ if self._open_since is None:
27
+ self._open_since = t
28
+ if t - self._open_since >= self.debounce:
29
+ self._open_since, self.closed = None, False
30
+ else:
31
+ self._open_since = None
32
+ return self.close_deg if self.closed else self.open_deg
33
+
34
+
35
+ def find_gripper_config(policy: Path, arg: str) -> dict | None:
36
+ """``auto``: look for gripper.json in the policy folder and up to three parents. ``raw``: none."""
37
+ if arg == "raw":
38
+ return None
39
+ cands = [Path(arg).expanduser()] if arg != "auto" else [d / "gripper.json" for d in [policy, *list(policy.parents)[:3]]]
40
+ for c in cands:
41
+ if c.is_file():
42
+ cfg = json.loads(c.read_text())
43
+ print(f"binary gripper ({c}): open {cfg['open_deg']} deg, closed {cfg['close_deg']} deg", flush=True)
44
+ return cfg
45
+ if arg != "auto":
46
+ raise FileNotFoundError(arg)
47
+ return None
@@ -0,0 +1,158 @@
1
+ """Policy wrappers: observation (camera images, measured joints) -> action chunk (H, 7) in degrees.
2
+
3
+ Both wrappers run on Apple silicon through PyTorch MPS.
4
+
5
+ pi0.5 notes (all measured on an M-series MacBook):
6
+ * Loading: the model constructor randomly initializes about 4B parameters (~80 s) that the checkpoint
7
+ then overwrites. We turn ``normal_`` and ``uniform_`` into no-ops while constructing.
8
+ * Precision: weights in bfloat16 under MPS autocast. The noise is created as float32 and MPS refuses
9
+ matmuls with mixed dtypes, so autocast is required.
10
+ * Empty camera slots: the base checkpoint has three image slots. With two cameras the third is a masked
11
+ blank image, which still costs a vision pass and 256 prefix tokens. Masked tokens take no part in
12
+ attention, so we drop them; outputs are identical and 10 denoising steps go from ~523 to ~453 ms.
13
+ * ``torch.compile``: each denoising step dispatches ~4300 small kernels and the CPU-side dispatch
14
+ dominates. Compiling ``denoise_step`` and the Gemma prefix forward brings 10 steps from ~412 to
15
+ ~282 ms. Compiling the SigLIP vision tower makes it slower, so it is left alone. First call compiles (~25 s).
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ import sys
21
+ from pathlib import Path
22
+
23
+ import cv2
24
+ import numpy as np
25
+
26
+
27
+ def prep_image(img_bgr, size, quality: int):
28
+ """Same path as recording: resize (INTER_AREA) -> JPEG -> decode -> RGB. Returns (RGB uint8, JPEG bytes)."""
29
+ small = cv2.resize(img_bgr, tuple(size), interpolation=cv2.INTER_AREA)
30
+ _, jpg = cv2.imencode(".jpg", small, [cv2.IMWRITE_JPEG_QUALITY, quality])
31
+ rgb = cv2.cvtColor(cv2.imdecode(jpg, cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
32
+ return rgb, jpg.reshape(-1)
33
+
34
+
35
+ def _to_obs(torch, imgs: dict, q, task: str, rename: dict | None = None):
36
+ obs = {"observation.state": torch.tensor(np.asarray(q, np.float32)).unsqueeze(0)}
37
+ for cam, img in imgs.items():
38
+ key = (rename or {}).get(cam, cam)
39
+ obs[f"observation.images.{key}"] = torch.from_numpy(img).permute(2, 0, 1).float().div(255).unsqueeze(0)
40
+ obs["task"] = [task]
41
+ return obs
42
+
43
+
44
+ class ACT:
45
+ kind = "act"
46
+ supports_rtc = False
47
+
48
+ def __init__(self, path: Path, device: str, task: str):
49
+ import torch
50
+ from lerobot.policies.act.modeling_act import ACTPolicy
51
+ from lerobot.policies.factory import make_pre_post_processors
52
+ self.torch, self.device, self.task = torch, device, task
53
+ self.pol = ACTPolicy.from_pretrained(path)
54
+ self.pol.config.device = device
55
+ self.pol.to(device).eval()
56
+ self.pre, self.post = make_pre_post_processors(
57
+ self.pol.config, pretrained_path=str(path), preprocessor_overrides={"device_processor": {"device": device}})
58
+ feats = self.pol.config.input_features
59
+ self.shapes = {k.removeprefix("observation.images."): tuple(feats[k].shape)
60
+ for k in feats if k.startswith("observation.images.")} # (3, H, W)
61
+ self.H = self.pol.config.chunk_size
62
+
63
+ def __call__(self, imgs: dict, q, **_) -> np.ndarray:
64
+ torch = self.torch
65
+ with torch.inference_mode():
66
+ a = self.post(self.pol.predict_action_chunk(self.pre(_to_obs(torch, imgs, q, self.task))))
67
+ if self.device == "mps":
68
+ torch.mps.synchronize()
69
+ return a.squeeze(0).float().cpu().numpy()
70
+
71
+
72
+ class PI05:
73
+ kind = "pi05"
74
+ supports_rtc = True
75
+ RENAME = {"top": "base_0_rgb", "wrist": "left_wrist_0_rgb"}
76
+
77
+ def __init__(self, path: Path, device: str, task: str, steps: int | None = None, rename: dict | None = None):
78
+ import torch
79
+ from lerobot.policies.factory import make_pre_post_processors
80
+ from lerobot.policies.pi05.modeling_pi05 import PI05Policy
81
+ self.torch, self.device, self.task = torch, device, task
82
+ self.rename = rename or self.RENAME
83
+ saved = {k: getattr(torch.Tensor, k) for k in ("normal_", "uniform_")}
84
+ try: # skip the random init that the checkpoint overwrites
85
+ for k in saved:
86
+ setattr(torch.Tensor, k, lambda self, *a, **kw: self)
87
+ self.pol = PI05Policy.from_pretrained(path)
88
+ finally:
89
+ for k, v in saved.items():
90
+ setattr(torch.Tensor, k, v)
91
+ self.pol.config.device = device
92
+ if steps:
93
+ self.pol.config.num_inference_steps = steps
94
+ self.pol.to(device=device, dtype=torch.bfloat16).eval()
95
+ self.pre, self.post = make_pre_post_processors(
96
+ self.pol.config, pretrained_path=str(path), preprocessor_overrides={"device_processor": {"device": device}})
97
+ self.shapes = None # pi0.5 resizes to 224 itself
98
+ self.H = self.pol.config.n_action_steps
99
+ self.rtc = False
100
+ self._skip_empty_cams()
101
+
102
+ def _skip_empty_cams(self):
103
+ model = self.pol.model
104
+ orig = model._embed_images
105
+
106
+ def embed(images, img_masks):
107
+ live = [j for j, m in enumerate(img_masks) if bool(m.any())]
108
+ if not live or len(live) == len(img_masks):
109
+ return orig(images, img_masks)
110
+ es = list(orig([images[j] for j in live], [img_masks[j] for j in live]))
111
+ it = iter(es)
112
+ return [next(it) if j in live else es[0][:, :0] for j in range(len(img_masks))]
113
+ model._embed_images = embed
114
+
115
+ def enable_compile(self):
116
+ torch, m = self.torch, self.pol.model
117
+ m.denoise_step = torch.compile(m.denoise_step, dynamic=False)
118
+ lm = m.paligemma_with_expert.paligemma.model.language_model
119
+ lm.forward = torch.compile(lm.forward, dynamic=False) # pi0.5 calls .forward directly
120
+
121
+ def enable_rtc(self, beta: float):
122
+ """Real-Time Chunking (Black et al., 2025) in LeRobot's guided mode."""
123
+ from lerobot.policies.rtc.configuration_rtc import RTCConfig
124
+ self.pol.config.rtc_config = RTCConfig(enabled=True, mode="guided", max_guidance_weight=beta,
125
+ execution_horizon=self.H)
126
+ self.pol.init_rtc_processor()
127
+ self.pol.requires_grad_(False) # guidance differentiates w.r.t. x_t only
128
+ self.rtc = True
129
+
130
+ def __call__(self, imgs: dict, q, leftover=None, delay: int = 0, horizon: int | None = None, return_raw=False):
131
+ """``leftover``: unexecuted rest of the current chunk in model space (for RTC guidance)."""
132
+ torch = self.torch
133
+ obs = _to_obs(torch, imgs, q, self.task, self.rename)
134
+ ac = (torch.autocast(self.device, dtype=torch.bfloat16) if self.device in ("mps", "cuda")
135
+ else torch.autocast("cpu", enabled=False))
136
+ kw = {}
137
+ if leftover is not None and len(leftover) > delay > 0:
138
+ kw = dict(prev_chunk_left_over=torch.from_numpy(np.ascontiguousarray(leftover, np.float32)).unsqueeze(0).to(self.device),
139
+ inference_delay=int(delay), execution_horizon=int(min(len(leftover), horizon or len(leftover))))
140
+ with (torch.no_grad() if self.rtc else torch.inference_mode()), ac: # guidance needs autograd
141
+ self.pol.reset()
142
+ raw = self.pol.predict_action_chunk(self.pre(obs), **kw)
143
+ raw_np = raw.float().cpu().numpy().copy()
144
+ a = self.post(raw)
145
+ if self.device == "mps":
146
+ torch.mps.synchronize()
147
+ a = a.float().cpu().numpy()
148
+ a, raw_np = (a[0] if a.ndim == 3 else a), (raw_np[0] if raw_np.ndim == 3 else raw_np)
149
+ return (a, raw_np) if return_raw else a
150
+
151
+
152
+ def load_policy(path: Path, device: str, task: str, steps: int | None = None):
153
+ typ = json.loads((Path(path) / "config.json").read_text()).get("type", "act")
154
+ if typ == "pi05":
155
+ return PI05(path, device, task, steps)
156
+ if typ == "act":
157
+ return ACT(path, device, task)
158
+ sys.exit(f"unsupported policy type {typ} (ACT and pi0.5 are supported)")