macbot 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- macbot/__init__.py +3 -0
- macbot/cameras/__init__.py +239 -0
- macbot/cameras/swift/uvc_stream.swift +83 -0
- macbot/cli.py +179 -0
- macbot/collect/__init__.py +0 -0
- macbot/collect/checks.py +36 -0
- macbot/collect/server.py +371 -0
- macbot/config.py +56 -0
- macbot/data/__init__.py +0 -0
- macbot/data/export.py +116 -0
- macbot/data/health.py +67 -0
- macbot/default.yaml +55 -0
- macbot/doctor.py +86 -0
- macbot/i18n.py +10 -0
- macbot/policy/__init__.py +0 -0
- macbot/policy/bench.py +28 -0
- macbot/policy/executors.py +222 -0
- macbot/policy/gripper.py +47 -0
- macbot/policy/models.py +158 -0
- macbot/policy/runner.py +345 -0
- macbot/robot/__init__.py +3 -0
- macbot/robot/fake.py +63 -0
- macbot/robot/gravity.py +108 -0
- macbot/robot/rebot.py +223 -0
- macbot/system.py +97 -0
- macbot/viz/__init__.py +0 -0
- macbot/viz/replay.py +113 -0
- macbot/viz/video.py +83 -0
- macbot/web/__init__.py +85 -0
- macbot/web/static/common.js +139 -0
- macbot/web/static/fonts/JetBrainsMono.ttf +0 -0
- macbot/web/static/fonts/OFL-JetBrainsMono.txt +93 -0
- macbot/web/static/logo-dark.svg +1 -0
- macbot/web/static/logo-mark.svg +1 -0
- macbot/web/static/logo.svg +1 -0
- macbot/web/static/theme.css +200 -0
- macbot/web/templates/_head.html +5 -0
- macbot/web/templates/collect.html +223 -0
- macbot/web/templates/replay.html +132 -0
- macbot/web/templates/run.html +124 -0
- macbot-0.1.0.dist-info/METADATA +183 -0
- macbot-0.1.0.dist-info/RECORD +46 -0
- macbot-0.1.0.dist-info/WHEEL +5 -0
- macbot-0.1.0.dist-info/entry_points.txt +2 -0
- macbot-0.1.0.dist-info/licenses/LICENSE +202 -0
- macbot-0.1.0.dist-info/top_level.txt +1 -0
macbot/__init__.py
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""Cameras for macOS.
|
|
2
|
+
|
|
3
|
+
* ``UVCCamera``: any UVC webcam, opened by its AVFoundation uniqueID through a tiny Swift helper
|
|
4
|
+
(compiled on first use). Opening by name or index can grab an iPhone Continuity Camera.
|
|
5
|
+
The helper crops, encodes JPEG and stamps the capture time.
|
|
6
|
+
* ``OrbbecColor``: color stream of an Orbbec Gemini 2 through ``pyorbbecsdk`` (needs sudo on macOS).
|
|
7
|
+
* ``FakeCamera``: moving test pattern for ``--fake`` runs.
|
|
8
|
+
|
|
9
|
+
All share one interface: ``latest() -> (BGR image, capture time)`` and ``close()``.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import struct
|
|
17
|
+
import subprocess
|
|
18
|
+
import threading
|
|
19
|
+
import time
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import cv2
|
|
23
|
+
import numpy as np
|
|
24
|
+
|
|
25
|
+
from ..system import give_back, user_home
|
|
26
|
+
|
|
27
|
+
SWIFT = Path(__file__).resolve().parent / "swift" / "uvc_stream.swift"
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class _Latest:
|
|
31
|
+
def __init__(self):
|
|
32
|
+
self.frame, self.stamp, self.lock, self.running = None, 0.0, threading.Lock(), True
|
|
33
|
+
|
|
34
|
+
def _put(self, img, stamp):
|
|
35
|
+
with self.lock:
|
|
36
|
+
self.frame, self.stamp = img, stamp
|
|
37
|
+
|
|
38
|
+
def latest(self):
|
|
39
|
+
with self.lock:
|
|
40
|
+
return self.frame, self.stamp
|
|
41
|
+
|
|
42
|
+
def age(self) -> float | None:
|
|
43
|
+
img, t = self.latest()
|
|
44
|
+
return None if img is None else time.time() - t
|
|
45
|
+
|
|
46
|
+
def _wait_first(self, name, alive=lambda: True, timeout=5.0):
|
|
47
|
+
t0 = time.time()
|
|
48
|
+
while self.frame is None:
|
|
49
|
+
if not alive() or time.time() - t0 > timeout:
|
|
50
|
+
raise RuntimeError(f"{name}: no frame within {timeout:.0f} s")
|
|
51
|
+
time.sleep(0.05)
|
|
52
|
+
time.sleep(1.0) # let auto exposure settle
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def uvc_helper() -> Path:
|
|
56
|
+
"""Compile the Swift helper once per source version into ~/.cache/macbot/bin."""
|
|
57
|
+
src = SWIFT.read_bytes()
|
|
58
|
+
out = user_home() / ".cache" / "macbot" / "bin" / f"uvc_stream_{hashlib.sha1(src).hexdigest()[:10]}"
|
|
59
|
+
if not out.exists():
|
|
60
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
61
|
+
print(f"compiling camera helper -> {out}", flush=True)
|
|
62
|
+
subprocess.run(["swiftc", "-O", str(SWIFT), "-o", str(out)], check=True)
|
|
63
|
+
for p in (out, out.parent, out.parent.parent):
|
|
64
|
+
give_back(p)
|
|
65
|
+
return out
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def list_cameras() -> list[dict]:
|
|
69
|
+
"""Video devices known to macOS: name, uniqueID, model."""
|
|
70
|
+
out = subprocess.run(["system_profiler", "SPCameraDataType", "-json"], capture_output=True, text=True,
|
|
71
|
+
timeout=15).stdout
|
|
72
|
+
return [{"name": c.get("_name", ""), "unique_id": c.get("spcamera_unique-id", ""),
|
|
73
|
+
"model": c.get("spcamera_model-id", "")} for c in json.loads(out or "{}").get("SPCameraDataType", [])]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def resolve_uid(unique_id: str) -> str:
|
|
77
|
+
"""A USB camera's uniqueID is location + vendor + product, so it changes with the USB port.
|
|
78
|
+
If the configured ID is absent, match on the last 8 hex digits (vendor and product)."""
|
|
79
|
+
try:
|
|
80
|
+
ids = [c["unique_id"] for c in list_cameras()]
|
|
81
|
+
except Exception:
|
|
82
|
+
return unique_id
|
|
83
|
+
if unique_id in ids:
|
|
84
|
+
return unique_id
|
|
85
|
+
tail = unique_id.lower()[-8:]
|
|
86
|
+
hit = [i for i in ids if i.lower().startswith("0x") and i.lower().endswith(tail)]
|
|
87
|
+
if len(hit) == 1:
|
|
88
|
+
print(f"camera {unique_id} not found; using {hit[0]} (same vendor and product, different USB port)", flush=True)
|
|
89
|
+
return hit[0]
|
|
90
|
+
return unique_id
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class UVCCamera(_Latest):
|
|
94
|
+
def __init__(self, unique_id: str, crop=None, fps: int = 30, name: str = "uvc"):
|
|
95
|
+
super().__init__()
|
|
96
|
+
crop = crop or [0, 0, 0, 0]
|
|
97
|
+
self.p = subprocess.Popen([str(uvc_helper()), resolve_uid(unique_id), *[str(int(v)) for v in crop], "0.92", str(fps)],
|
|
98
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
99
|
+
threading.Thread(target=self._loop, daemon=True).start()
|
|
100
|
+
try:
|
|
101
|
+
self._wait_first(name, alive=lambda: self.p.poll() is None)
|
|
102
|
+
except RuntimeError as e:
|
|
103
|
+
err = self.p.stderr.read().decode(errors="replace").strip() if self.p.poll() is not None else ""
|
|
104
|
+
self.close()
|
|
105
|
+
raise RuntimeError(f"{e} {err}") from None
|
|
106
|
+
|
|
107
|
+
def _read(self, n):
|
|
108
|
+
b = b""
|
|
109
|
+
while len(b) < n:
|
|
110
|
+
c = self.p.stdout.read(n - len(b))
|
|
111
|
+
if not c:
|
|
112
|
+
raise EOFError
|
|
113
|
+
b += c
|
|
114
|
+
return b
|
|
115
|
+
|
|
116
|
+
def _loop(self):
|
|
117
|
+
try:
|
|
118
|
+
while self.running:
|
|
119
|
+
t, n = struct.unpack("<dI", self._read(12))
|
|
120
|
+
img = cv2.imdecode(np.frombuffer(self._read(n), np.uint8), cv2.IMREAD_COLOR)
|
|
121
|
+
if img is not None:
|
|
122
|
+
self._put(img, t)
|
|
123
|
+
except (EOFError, ValueError, OSError):
|
|
124
|
+
pass
|
|
125
|
+
|
|
126
|
+
def close(self):
|
|
127
|
+
self.running = False
|
|
128
|
+
if self.p.poll() is None:
|
|
129
|
+
self.p.terminate()
|
|
130
|
+
try:
|
|
131
|
+
self.p.wait(timeout=2)
|
|
132
|
+
except subprocess.TimeoutExpired:
|
|
133
|
+
self.p.kill()
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _orbbec_to_bgr(f):
|
|
137
|
+
from pyorbbecsdk import OBFormat
|
|
138
|
+
w, h, fmt = f.get_width(), f.get_height(), f.get_format()
|
|
139
|
+
buf = np.asanyarray(f.get_data())
|
|
140
|
+
if fmt == OBFormat.MJPG:
|
|
141
|
+
return cv2.imdecode(buf, cv2.IMREAD_COLOR)
|
|
142
|
+
if fmt == OBFormat.RGB:
|
|
143
|
+
return cv2.cvtColor(buf.reshape(h, w, 3), cv2.COLOR_RGB2BGR)
|
|
144
|
+
if fmt == OBFormat.BGR:
|
|
145
|
+
return buf.reshape(h, w, 3)
|
|
146
|
+
if fmt == OBFormat.YUYV:
|
|
147
|
+
return cv2.cvtColor(buf.reshape(h, w, 2), cv2.COLOR_YUV2BGR_YUYV)
|
|
148
|
+
if fmt == OBFormat.UYVY:
|
|
149
|
+
return cv2.cvtColor(buf.reshape(h, w, 2), cv2.COLOR_YUV2BGR_UYVY)
|
|
150
|
+
raise RuntimeError(f"unhandled color format {fmt}")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class OrbbecColor(_Latest):
|
|
154
|
+
def __init__(self, expect_size=(1280, 720), name: str = "orbbec"):
|
|
155
|
+
super().__init__()
|
|
156
|
+
from pyorbbecsdk import Config, OBSensorType, Pipeline
|
|
157
|
+
self.pipe = Pipeline()
|
|
158
|
+
prof = self.pipe.get_stream_profile_list(OBSensorType.COLOR_SENSOR).get_default_video_stream_profile()
|
|
159
|
+
if (prof.get_width(), prof.get_height()) != tuple(expect_size):
|
|
160
|
+
raise RuntimeError(f"{name}: color stream is {prof.get_width()}x{prof.get_height()}, config expects {expect_size}")
|
|
161
|
+
cfg = Config()
|
|
162
|
+
cfg.enable_stream(prof)
|
|
163
|
+
self.pipe.start(cfg)
|
|
164
|
+
self.th = threading.Thread(target=self._loop, daemon=True)
|
|
165
|
+
self.th.start()
|
|
166
|
+
try:
|
|
167
|
+
self._wait_first(name)
|
|
168
|
+
except RuntimeError:
|
|
169
|
+
self.close()
|
|
170
|
+
raise
|
|
171
|
+
|
|
172
|
+
def _loop(self):
|
|
173
|
+
while self.running:
|
|
174
|
+
fs = self.pipe.wait_for_frames(200)
|
|
175
|
+
c = fs.get_color_frame() if fs is not None else None
|
|
176
|
+
img = _orbbec_to_bgr(c) if c is not None else None
|
|
177
|
+
if img is not None:
|
|
178
|
+
self._put(img, time.time())
|
|
179
|
+
|
|
180
|
+
def close(self):
|
|
181
|
+
self.running = False
|
|
182
|
+
self.th.join(timeout=1)
|
|
183
|
+
self.pipe.stop()
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
class FakeCamera(_Latest):
|
|
187
|
+
"""30 fps test pattern: a dark tray and an orange ball moving left and right."""
|
|
188
|
+
|
|
189
|
+
def __init__(self, kind="top", size=(985, 940)):
|
|
190
|
+
super().__init__()
|
|
191
|
+
self.kind, self.size = kind, tuple(size)
|
|
192
|
+
threading.Thread(target=self._loop, daemon=True).start()
|
|
193
|
+
self._wait_first(f"fake {kind}", timeout=2)
|
|
194
|
+
|
|
195
|
+
def _loop(self):
|
|
196
|
+
w, h = self.size
|
|
197
|
+
while self.running:
|
|
198
|
+
t = time.time()
|
|
199
|
+
img = np.full((h, w, 3), (220, 244, 255) if self.kind == "top" else (76, 42, 43), np.uint8)
|
|
200
|
+
cv2.rectangle(img, (w // 10, h // 6), (w * 9 // 10, h * 2 // 3), (46, 26, 27), -1)
|
|
201
|
+
cv2.circle(img, (int(w / 2 + w / 4 * math.sin(t)), h // 2), h // 12, (26, 122, 255), -1)
|
|
202
|
+
cv2.putText(img, f"fake {self.kind}", (20, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.2, (60, 201, 255), 3)
|
|
203
|
+
self._put(img, t)
|
|
204
|
+
time.sleep(1 / 30)
|
|
205
|
+
|
|
206
|
+
def close(self):
|
|
207
|
+
self.running = False
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _source_size(c: dict) -> tuple[int, int]:
|
|
211
|
+
if c.get("crop"):
|
|
212
|
+
x0, y0, x1, y1 = c["crop"]
|
|
213
|
+
return x1 - x0, y1 - y0
|
|
214
|
+
return tuple(c.get("expect_size") or (1280, 720))
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def open_cameras(cams_cfg: dict, fps: int, fake: bool) -> dict:
|
|
218
|
+
"""Open every camera in the ``cameras`` config section. Closes what was opened if one fails."""
|
|
219
|
+
out = {}
|
|
220
|
+
try:
|
|
221
|
+
for name, c in cams_cfg.items():
|
|
222
|
+
if fake:
|
|
223
|
+
out[name] = FakeCamera(name, _source_size(c))
|
|
224
|
+
elif c["type"] == "uvc":
|
|
225
|
+
out[name] = UVCCamera(c["unique_id"], c.get("crop"), fps, name)
|
|
226
|
+
elif c["type"] == "orbbec":
|
|
227
|
+
out[name] = OrbbecColor(tuple(c.get("expect_size", (1280, 720))), name)
|
|
228
|
+
else:
|
|
229
|
+
raise ValueError(f"camera {name}: unknown type {c['type']}")
|
|
230
|
+
except Exception:
|
|
231
|
+
for cam in out.values():
|
|
232
|
+
cam.close()
|
|
233
|
+
raise
|
|
234
|
+
return out
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def camera_info(cams_cfg: dict) -> dict:
|
|
238
|
+
return {k: {"type": c["type"], "unique_id": c.get("unique_id"), "crop": c.get("crop"), "size": c["save_size"]}
|
|
239
|
+
for k, c in cams_cfg.items()}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Stream a UVC camera opened by its AVFoundation uniqueID as cropped JPEG frames on stdout.
|
|
2
|
+
// Opening by uniqueID avoids grabbing a Continuity Camera (iPhone) by accident.
|
|
3
|
+
//
|
|
4
|
+
// Frame format: 8-byte little-endian Double (capture time, Unix seconds)
|
|
5
|
+
// + 4-byte little-endian UInt32 (JPEG length) + JPEG bytes.
|
|
6
|
+
// Usage: uvc_stream <uniqueID> <x0> <y0> <x1> <y1> [jpeg quality 0-1, default 0.9] [fps, default 30]
|
|
7
|
+
// x1 = 0 streams the full frame. Crop coordinates are top-left based pixels (as in OpenCV).
|
|
8
|
+
// The process exits when the parent closes the pipe (SIGPIPE) or on SIGTERM.
|
|
9
|
+
import AVFoundation
|
|
10
|
+
import CoreImage
|
|
11
|
+
import Foundation
|
|
12
|
+
|
|
13
|
+
func fail(_ msg: String, _ code: Int32) -> Never {
|
|
14
|
+
FileHandle.standardError.write((msg + "\n").data(using: .utf8)!)
|
|
15
|
+
exit(code)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
let a = CommandLine.arguments
|
|
19
|
+
guard a.count >= 6, let x0 = Int(a[2]), let y0 = Int(a[3]), let x1 = Int(a[4]), let y1 = Int(a[5]) else {
|
|
20
|
+
fail("usage: uvc_stream <uniqueID> <x0> <y0> <x1> <y1> [quality] [fps]", 2)
|
|
21
|
+
}
|
|
22
|
+
let quality = a.count > 6 ? Double(a[6])! : 0.9
|
|
23
|
+
let fps = a.count > 7 ? Int32(a[7])! : 30
|
|
24
|
+
guard let dev = AVCaptureDevice(uniqueID: a[1]) else { fail("camera not found: \(a[1])", 1) }
|
|
25
|
+
|
|
26
|
+
final class Out: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate {
|
|
27
|
+
let x0: Int, y0: Int, x1: Int, y1: Int, quality: Double
|
|
28
|
+
let ctx = CIContext()
|
|
29
|
+
let cs = CGColorSpace(name: CGColorSpace.sRGB)!
|
|
30
|
+
var checked = false
|
|
31
|
+
init(_ x0: Int, _ y0: Int, _ x1: Int, _ y1: Int, _ q: Double) {
|
|
32
|
+
self.x0 = x0; self.y0 = y0; self.x1 = x1; self.y1 = y1; self.quality = q
|
|
33
|
+
}
|
|
34
|
+
func captureOutput(_ o: AVCaptureOutput, didOutput sb: CMSampleBuffer, from c: AVCaptureConnection) {
|
|
35
|
+
let t = Date().timeIntervalSince1970
|
|
36
|
+
guard let pb = CMSampleBufferGetImageBuffer(sb) else { return }
|
|
37
|
+
let w = CVPixelBufferGetWidth(pb), h = CVPixelBufferGetHeight(pb)
|
|
38
|
+
let full = x1 == 0
|
|
39
|
+
if !checked {
|
|
40
|
+
checked = true
|
|
41
|
+
if !full && (x1 > w || y1 > h) { fail("frame is \(w)x\(h), crop box exceeds it", 3) }
|
|
42
|
+
}
|
|
43
|
+
var img = CIImage(cvPixelBuffer: pb)
|
|
44
|
+
if !full { // CoreImage has its origin at the bottom left
|
|
45
|
+
let r = CGRect(x: x0, y: h - y1, width: x1 - x0, height: y1 - y0)
|
|
46
|
+
img = img.cropped(to: r).transformed(by: CGAffineTransform(translationX: -r.minX, y: -r.minY))
|
|
47
|
+
}
|
|
48
|
+
let opts = [CIImageRepresentationOption(rawValue: kCGImageDestinationLossyCompressionQuality as String): quality]
|
|
49
|
+
guard let jpg = ctx.jpegRepresentation(of: img, colorSpace: cs, options: opts) else { return }
|
|
50
|
+
var head = Data(count: 12)
|
|
51
|
+
head.withUnsafeMutableBytes { p in
|
|
52
|
+
p.storeBytes(of: t.bitPattern.littleEndian, toByteOffset: 0, as: UInt64.self)
|
|
53
|
+
p.storeBytes(of: UInt32(jpg.count).littleEndian, toByteOffset: 8, as: UInt32.self)
|
|
54
|
+
}
|
|
55
|
+
FileHandle.standardOutput.write(head + jpg)
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
let s = AVCaptureSession()
|
|
60
|
+
if s.canSetSessionPreset(.hd1920x1080) { s.sessionPreset = .hd1920x1080 }
|
|
61
|
+
s.addInput(try! AVCaptureDeviceInput(device: dev))
|
|
62
|
+
let o = AVCaptureVideoDataOutput()
|
|
63
|
+
o.alwaysDiscardsLateVideoFrames = true
|
|
64
|
+
o.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA]
|
|
65
|
+
let out = Out(x0, y0, x1, y1, quality)
|
|
66
|
+
o.setSampleBufferDelegate(out, queue: DispatchQueue(label: "cam"))
|
|
67
|
+
s.addOutput(o)
|
|
68
|
+
s.startRunning()
|
|
69
|
+
do {
|
|
70
|
+
// Many UVC cameras default to 25 fps. Lock the rate using the exact duration the format reports
|
|
71
|
+
// (the 30 fps entry is 1000000/30000030 s, not 1/30; anything else throws).
|
|
72
|
+
try dev.lockForConfiguration()
|
|
73
|
+
if let r = dev.activeFormat.videoSupportedFrameRateRanges.min(by: {
|
|
74
|
+
abs($0.maxFrameRate - Double(fps)) < abs($1.maxFrameRate - Double(fps)) }) {
|
|
75
|
+
dev.activeVideoMinFrameDuration = r.minFrameDuration
|
|
76
|
+
dev.activeVideoMaxFrameDuration = r.minFrameDuration
|
|
77
|
+
}
|
|
78
|
+
dev.unlockForConfiguration()
|
|
79
|
+
} catch {
|
|
80
|
+
FileHandle.standardError.write("could not set frame rate: \(error)\n".data(using: .utf8)!)
|
|
81
|
+
}
|
|
82
|
+
signal(SIGTERM) { _ in exit(0) }
|
|
83
|
+
RunLoop.main.run()
|
macbot/cli.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""Command line entry point: ``macbot <command>``.
|
|
2
|
+
|
|
3
|
+
macbot doctor check the Mac: Python packages, MPS, PCAN driver, cameras, sudo
|
|
4
|
+
macbot cameras list video devices with their uniqueIDs
|
|
5
|
+
macbot joints read leader and follower joint angles without powering anything
|
|
6
|
+
macbot collect [--fake] teleoperate and record episodes (web page)
|
|
7
|
+
macbot export convert recorded episodes to a LeRobot dataset
|
|
8
|
+
macbot health per-episode tracking and timing report of recorded data
|
|
9
|
+
macbot run --policy DIR run a trained ACT or pi0.5 policy on the robot (web page)
|
|
10
|
+
macbot replay browse recorded rollouts in the browser
|
|
11
|
+
macbot video ROLLOUT export a rollout as a side-by-side MP4
|
|
12
|
+
macbot bench --policy DIR time policy inference on this Mac
|
|
13
|
+
|
|
14
|
+
Robot commands need ``sudo -E`` on macOS (the Orbbec SDK requires root). Everything that does not
|
|
15
|
+
touch the hardware runs as a normal user, and ``--fake`` simulates robot and cameras.
|
|
16
|
+
"""
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import argparse
|
|
20
|
+
import sys
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _cfg(args):
|
|
25
|
+
from .config import load
|
|
26
|
+
return load(args.config)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def cmd_doctor(args):
|
|
30
|
+
from .doctor import run
|
|
31
|
+
return run(_cfg(args))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def cmd_cameras(args):
|
|
35
|
+
from .cameras import list_cameras
|
|
36
|
+
cams = list_cameras()
|
|
37
|
+
if not cams:
|
|
38
|
+
print("no video devices found")
|
|
39
|
+
for c in cams:
|
|
40
|
+
print(f"{c['name']:<36s} {c['unique_id']:<24s} {c['model']}")
|
|
41
|
+
print("\nPut the uniqueID of your scene camera into cameras.<name>.unique_id.")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def cmd_joints(args):
|
|
45
|
+
from .system import preload_pcan
|
|
46
|
+
preload_pcan()
|
|
47
|
+
from .config import JOINTS
|
|
48
|
+
from .robot.rebot import make_leader, read_joints_passive
|
|
49
|
+
cfg = _cfg(args)
|
|
50
|
+
leader = make_leader(leader_id=cfg["robot"].get("leader_id", "rebot_arm_102_leader"))
|
|
51
|
+
leader.connect(calibrate=False) # leader: unlock (torque off) and reset turn counters; it does not move
|
|
52
|
+
act = leader.get_action()
|
|
53
|
+
leader.disconnect()
|
|
54
|
+
fol = read_joints_passive(cfg["robot"])
|
|
55
|
+
print(f"{'joint':14s} {'leader':>8s} {'follower':>9s} {'diff':>8s}")
|
|
56
|
+
for n in JOINTS:
|
|
57
|
+
a, f = act[f"{n}.pos"], fol[n]
|
|
58
|
+
print(f"{n:14s} {a:8.1f} {f:9.1f} {a - f:+8.1f}")
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def cmd_collect(args):
|
|
62
|
+
if not args.fake:
|
|
63
|
+
from .system import preload_pcan
|
|
64
|
+
preload_pcan()
|
|
65
|
+
from .collect.server import run
|
|
66
|
+
run(_cfg(args), fake=args.fake, port=args.port, out=args.out)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def cmd_export(args):
|
|
70
|
+
from .data.export import run
|
|
71
|
+
run(_cfg(args), args)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def cmd_health(args):
|
|
75
|
+
from .data.health import run
|
|
76
|
+
run(_cfg(args), args)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def cmd_run(args):
|
|
80
|
+
if not args.fake:
|
|
81
|
+
from .system import preload_pcan
|
|
82
|
+
preload_pcan()
|
|
83
|
+
from .policy.runner import run
|
|
84
|
+
run(_cfg(args), args)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def cmd_replay(args):
|
|
88
|
+
from .viz.replay import run
|
|
89
|
+
run(_cfg(args), args)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def cmd_video(args):
|
|
93
|
+
from .viz.video import run
|
|
94
|
+
run(args)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def cmd_bench(args):
|
|
98
|
+
from .policy.bench import run
|
|
99
|
+
run(_cfg(args), args)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def main(argv=None):
|
|
103
|
+
ap = argparse.ArgumentParser(prog="macbot", description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
104
|
+
ap.add_argument("--config", help="YAML file overriding the built-in defaults")
|
|
105
|
+
sub = ap.add_subparsers(dest="cmd", required=True, metavar="command")
|
|
106
|
+
|
|
107
|
+
sub.add_parser("doctor", help="check this Mac").set_defaults(fn=cmd_doctor)
|
|
108
|
+
sub.add_parser("cameras", help="list video devices and uniqueIDs").set_defaults(fn=cmd_cameras)
|
|
109
|
+
sub.add_parser("joints", help="read joint angles without powering motors").set_defaults(fn=cmd_joints)
|
|
110
|
+
|
|
111
|
+
p = sub.add_parser("collect", help="teleoperate and record (web page)")
|
|
112
|
+
p.add_argument("--fake", action="store_true", help="simulate robot, leader and cameras")
|
|
113
|
+
p.add_argument("--port", type=int)
|
|
114
|
+
p.add_argument("--out", help="episode folder (default <data_root>/raw/<task>)")
|
|
115
|
+
p.set_defaults(fn=cmd_collect)
|
|
116
|
+
|
|
117
|
+
p = sub.add_parser("export", help="recorded episodes -> LeRobot dataset")
|
|
118
|
+
p.add_argument("--data", help="episode folder (default <data_root>/raw/<task>)")
|
|
119
|
+
p.add_argument("--out", help="dataset folder (default <data_root>/datasets/<task>[_bin])")
|
|
120
|
+
p.add_argument("--repo-id")
|
|
121
|
+
p.add_argument("--gripper", choices=["raw", "binary"], default="raw", help="gripper action: leader angle, or 0/1")
|
|
122
|
+
p.add_argument("--close-above", type=float, default=-40.0, help="binary: leader angle above this = closed")
|
|
123
|
+
p.add_argument("--open-below", type=float, default=-80.0, help="binary: once closed, below this = open")
|
|
124
|
+
p.add_argument("--open-deg", type=float, default=-150.0, help="binary: gripper angle commanded for 'open'")
|
|
125
|
+
p.add_argument("--close-deg", type=float, default=0.0, help="binary: gripper angle commanded for 'closed'")
|
|
126
|
+
p.add_argument("--skip-warnings", action="store_true", help="also skip episodes with warnings")
|
|
127
|
+
p.add_argument("--images", action="store_true", help="store PNG frames instead of video (debugging)")
|
|
128
|
+
p.add_argument("--overwrite", action="store_true")
|
|
129
|
+
p.set_defaults(fn=cmd_export)
|
|
130
|
+
|
|
131
|
+
p = sub.add_parser("health", help="tracking and timing report of recorded episodes")
|
|
132
|
+
p.add_argument("--data", help="episode folder (default <data_root>/raw/<task>)")
|
|
133
|
+
p.add_argument("--split", type=int, help="compare episodes before / from this index (e.g. after a repair)")
|
|
134
|
+
p.set_defaults(fn=cmd_health)
|
|
135
|
+
|
|
136
|
+
p = sub.add_parser("run", help="run a policy on the robot (web page)")
|
|
137
|
+
p.add_argument("--policy", required=True, help="pretrained_model folder (ACT or pi0.5)")
|
|
138
|
+
p.add_argument("--mode", choices=["auto", "chunk", "ensemble", "async", "rtc"], default="auto",
|
|
139
|
+
help="auto: ACT -> ensemble, pi0.5 -> rtc")
|
|
140
|
+
p.add_argument("--k", type=int, default=30, help="chunk mode: steps executed per chunk")
|
|
141
|
+
p.add_argument("--te-m", type=float, default=0.01, help="ensemble: weight exp(m * age), as in ACT")
|
|
142
|
+
p.add_argument("--te-hz", type=float, default=30.0, help="ensemble: max inference rate")
|
|
143
|
+
p.add_argument("--rtc-beta", type=float, default=10.0, help="rtc: max guidance weight")
|
|
144
|
+
p.add_argument("--rtc-horizon", type=int, default=20, help="rtc: guided prefix length in policy steps")
|
|
145
|
+
p.add_argument("--gripper", default="auto", help="auto (find gripper.json), raw, or a path to gripper.json")
|
|
146
|
+
p.add_argument("--grip-debounce", type=float, default=0.6, help="binary gripper: seconds of 'open' before opening")
|
|
147
|
+
p.add_argument("--steps", type=int, help="pi0.5 denoising steps (default from checkpoint)")
|
|
148
|
+
p.add_argument("--no-compile", action="store_true", help="pi0.5: skip torch.compile")
|
|
149
|
+
p.add_argument("--speed", type=float, help="0.1-1.0, default from config")
|
|
150
|
+
p.add_argument("--device", default="mps")
|
|
151
|
+
p.add_argument("--port", type=int)
|
|
152
|
+
p.add_argument("--fake", action="store_true", help="simulate robot and cameras (policy runs for real)")
|
|
153
|
+
p.add_argument("--autostart", type=float, default=0, help="with --fake: run this many seconds, save, quit")
|
|
154
|
+
p.set_defaults(fn=cmd_run)
|
|
155
|
+
|
|
156
|
+
p = sub.add_parser("replay", help="browse recorded rollouts")
|
|
157
|
+
p.add_argument("--dir", help="rollout folder (default <data_root>/rollouts/<task>)")
|
|
158
|
+
p.add_argument("--port", type=int, default=8768)
|
|
159
|
+
p.set_defaults(fn=cmd_replay)
|
|
160
|
+
|
|
161
|
+
p = sub.add_parser("video", help="rollout -> side-by-side MP4")
|
|
162
|
+
p.add_argument("rollout")
|
|
163
|
+
p.add_argument("--out")
|
|
164
|
+
p.add_argument("--height", type=int, default=540)
|
|
165
|
+
p.set_defaults(fn=cmd_video)
|
|
166
|
+
|
|
167
|
+
p = sub.add_parser("bench", help="time policy inference")
|
|
168
|
+
p.add_argument("--policy", required=True)
|
|
169
|
+
p.add_argument("--device", default="mps")
|
|
170
|
+
p.add_argument("-n", type=int, default=20)
|
|
171
|
+
p.add_argument("--no-compile", action="store_true")
|
|
172
|
+
p.set_defaults(fn=cmd_bench)
|
|
173
|
+
|
|
174
|
+
args = ap.parse_args(argv)
|
|
175
|
+
return args.fn(args)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
if __name__ == "__main__":
|
|
179
|
+
sys.exit(main())
|
|
File without changes
|
macbot/collect/checks.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""Quality checks run on every saved episode."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
|
|
6
|
+
from ..config import ARM
|
|
7
|
+
from ..i18n import tr
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def check_episode(d: dict, cams, fps: int, checks: dict, min_s: float) -> dict:
|
|
11
|
+
"""``d`` holds arrays ``qpos``, ``late`` and ``<cam>_age`` / ``<cam>_repeat`` per camera.
|
|
12
|
+
Returns ``{ok, errors, warnings, stats}``; errors block export, warnings are shown."""
|
|
13
|
+
n = len(d["qpos"])
|
|
14
|
+
errors, warnings, stats = [], [], {}
|
|
15
|
+
if n / fps < min_s:
|
|
16
|
+
errors.append(tr(f"only {n / fps:.1f} s", f"只有 {n / fps:.1f} 秒"))
|
|
17
|
+
for k in cams:
|
|
18
|
+
age95 = float(np.percentile(d[f"{k}_age"], 95)) if n else 0.0
|
|
19
|
+
rep = float(np.mean(d[f"{k}_repeat"])) if n else 0.0
|
|
20
|
+
stats[f"{k}_age_p95_ms"] = round(age95 * 1e3, 1)
|
|
21
|
+
stats[f"{k}_repeat_ratio"] = round(rep, 3)
|
|
22
|
+
if age95 > checks["max_cam_age_s"]:
|
|
23
|
+
warnings.append(tr(f"{k} frame age p95 {age95 * 1e3:.0f} ms", f"{k} 画面延迟 p95 {age95 * 1e3:.0f} ms"))
|
|
24
|
+
if rep > checks["max_repeat_ratio"]:
|
|
25
|
+
warnings.append(tr(f"{k} repeated frames {rep:.0%}", f"{k} 画面重复 {rep:.0%}"))
|
|
26
|
+
if n > 1:
|
|
27
|
+
step = np.abs(np.diff(np.asarray(d["qpos"])[:, :6], axis=0)).max(axis=0)
|
|
28
|
+
stats["max_joint_step_deg"] = round(float(step.max()), 1)
|
|
29
|
+
if step.max() > checks["max_joint_step_deg"]:
|
|
30
|
+
warnings.append(tr(f"{ARM[int(step.argmax())]} jumped {step.max():.0f} deg in one frame",
|
|
31
|
+
f"{ARM[int(step.argmax())]} 一帧跳 {step.max():.0f}°"))
|
|
32
|
+
late = float(np.mean(d["late"])) if n else 0.0
|
|
33
|
+
stats["late_ratio"] = round(late, 3)
|
|
34
|
+
if late > checks["max_late_ratio"]:
|
|
35
|
+
warnings.append(tr(f"late samples {late:.0%}", f"采样迟到 {late:.0%}"))
|
|
36
|
+
return {"ok": not errors, "errors": errors, "warnings": warnings, "stats": stats}
|