rcs-zed 0.7.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rcs_zed/__init__.py +1 -0
- rcs_zed/__main__.py +194 -0
- rcs_zed/camera.py +411 -0
- rcs_zed/utils.py +30 -0
- rcs_zed-0.7.3.dist-info/METADATA +87 -0
- rcs_zed-0.7.3.dist-info/RECORD +8 -0
- rcs_zed-0.7.3.dist-info/WHEEL +5 -0
- rcs_zed-0.7.3.dist-info/top_level.txt +1 -0
rcs_zed/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.6.3"
|
rcs_zed/__main__.py
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from time import monotonic
|
|
4
|
+
|
|
5
|
+
import cv2
|
|
6
|
+
import typer
|
|
7
|
+
from rcs_zed.camera import ZEDCameraSet
|
|
8
|
+
|
|
9
|
+
from rcs import common
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
zed_app = typer.Typer(help="CLI tools for the ZED camera module of rcs.")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _display_frame(window_name: str, frame):
|
|
16
|
+
cv2.imshow(window_name, frame.camera.color.data[:, :, ::-1])
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _resolve_serial(serial: str | None) -> str:
|
|
20
|
+
if serial is not None:
|
|
21
|
+
return serial
|
|
22
|
+
|
|
23
|
+
try:
|
|
24
|
+
devices = ZEDCameraSet.enumerate_connected_devices()
|
|
25
|
+
except RuntimeError as exc:
|
|
26
|
+
msg = str(exc)
|
|
27
|
+
raise typer.BadParameter(msg) from exc
|
|
28
|
+
if len(devices) == 0:
|
|
29
|
+
msg = "No ZED devices connected."
|
|
30
|
+
raise typer.BadParameter(msg)
|
|
31
|
+
return next(iter(devices))
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _create_rgb_camera(serial: str, width: int, height: int, fps: int) -> ZEDCameraSet:
|
|
35
|
+
return ZEDCameraSet(
|
|
36
|
+
cameras={
|
|
37
|
+
"viewer": common.BaseCameraConfig(
|
|
38
|
+
identifier=serial,
|
|
39
|
+
resolution_width=width,
|
|
40
|
+
resolution_height=height,
|
|
41
|
+
frame_rate=fps,
|
|
42
|
+
)
|
|
43
|
+
},
|
|
44
|
+
enable_depth=False,
|
|
45
|
+
enable_imu=False,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@zed_app.command()
|
|
50
|
+
def serials():
|
|
51
|
+
"""Reads out the serial numbers and models of connected ZED devices via the SDK."""
|
|
52
|
+
try:
|
|
53
|
+
devices = ZEDCameraSet.enumerate_connected_devices()
|
|
54
|
+
except RuntimeError as exc:
|
|
55
|
+
typer.secho(str(exc), fg=typer.colors.YELLOW, err=True)
|
|
56
|
+
return
|
|
57
|
+
if len(devices) == 0:
|
|
58
|
+
typer.secho("No ZED devices connected or the ZED SDK is not available.", fg=typer.colors.YELLOW, err=True)
|
|
59
|
+
return
|
|
60
|
+
typer.echo("Connected devices:")
|
|
61
|
+
for device in devices.values():
|
|
62
|
+
typer.echo(f" {device.model}: {device.serial} (imu={device.has_imu})")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _format_intrinsics(name: str, matrix) -> str:
|
|
66
|
+
fx, fy, cx, cy = float(matrix[0, 0]), float(matrix[1, 1]), float(matrix[0, 2]), float(matrix[1, 2])
|
|
67
|
+
lines = [
|
|
68
|
+
f"{name}:",
|
|
69
|
+
f" fx={fx:.6f} fy={fy:.6f} cx={cx:.6f} cy={cy:.6f}",
|
|
70
|
+
f" K=\n{matrix}",
|
|
71
|
+
]
|
|
72
|
+
return "\n".join(lines)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@zed_app.command()
|
|
76
|
+
def intrinsics(
|
|
77
|
+
serial: str | None = typer.Argument(None, help="Optional ZED serial number. Uses the first device if omitted."),
|
|
78
|
+
width: int = typer.Option(1280, help="Requested capture width."),
|
|
79
|
+
height: int = typer.Option(720, help="Requested capture height."),
|
|
80
|
+
fps: int = typer.Option(30, help="Requested capture frame rate."),
|
|
81
|
+
right: bool = typer.Option(False, "--right", help="Also print right-camera intrinsics."),
|
|
82
|
+
):
|
|
83
|
+
"""Open a ZED camera and print its RGB intrinsics for the requested resolution."""
|
|
84
|
+
serial = _resolve_serial(serial)
|
|
85
|
+
config = common.BaseCameraConfig(
|
|
86
|
+
identifier=serial,
|
|
87
|
+
resolution_width=width,
|
|
88
|
+
resolution_height=height,
|
|
89
|
+
frame_rate=fps,
|
|
90
|
+
)
|
|
91
|
+
try:
|
|
92
|
+
handle = ZEDCameraSet.open_camera(
|
|
93
|
+
config,
|
|
94
|
+
enable_depth=False,
|
|
95
|
+
enable_imu=False,
|
|
96
|
+
include_right=right,
|
|
97
|
+
)
|
|
98
|
+
except Exception as exc:
|
|
99
|
+
msg = f"Could not start ZED camera {serial}: {exc}"
|
|
100
|
+
raise typer.BadParameter(msg) from exc
|
|
101
|
+
|
|
102
|
+
try:
|
|
103
|
+
typer.echo(f"serial={handle.device_info.serial} model={handle.device_info.model} resolution={width}x{height}")
|
|
104
|
+
typer.echo(_format_intrinsics("left", handle.color_intrinsics))
|
|
105
|
+
if right and handle.right_color_intrinsics is not None:
|
|
106
|
+
typer.echo(_format_intrinsics("right", handle.right_color_intrinsics))
|
|
107
|
+
finally:
|
|
108
|
+
handle.close()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@zed_app.command("rgb-view")
|
|
112
|
+
def rgb_view(
|
|
113
|
+
serial: str | None = typer.Argument(None, help="Optional ZED serial number. Uses the first device if omitted."),
|
|
114
|
+
width: int = typer.Option(1280, help="Requested capture width."),
|
|
115
|
+
height: int = typer.Option(720, help="Requested capture height."),
|
|
116
|
+
fps: int = typer.Option(30, help="Requested capture frame rate."),
|
|
117
|
+
window_name: str = typer.Option("ZED RGB", help="OpenCV window title."),
|
|
118
|
+
):
|
|
119
|
+
"""Open a live RGB window using the RCS ZED camera interface."""
|
|
120
|
+
serial = _resolve_serial(serial)
|
|
121
|
+
camera = _create_rgb_camera(serial, width, height, fps)
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
camera.open()
|
|
125
|
+
except Exception as exc:
|
|
126
|
+
msg = f"Could not start ZED camera {serial}: {exc}"
|
|
127
|
+
raise typer.BadParameter(msg) from exc
|
|
128
|
+
|
|
129
|
+
logger.info("Streaming RGB from ZED %s. Press 'q' to quit.", serial)
|
|
130
|
+
try:
|
|
131
|
+
while True:
|
|
132
|
+
frame = camera.poll_frame("viewer")
|
|
133
|
+
_display_frame(window_name, frame)
|
|
134
|
+
if cv2.waitKey(1) & 0xFF == ord("q"):
|
|
135
|
+
break
|
|
136
|
+
finally:
|
|
137
|
+
camera.close()
|
|
138
|
+
cv2.destroyAllWindows()
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@zed_app.command("rgb-snapshot")
|
|
142
|
+
def rgb_snapshot(
|
|
143
|
+
serial: str | None = typer.Argument(None, help="Optional ZED serial number. Uses the first device if omitted."),
|
|
144
|
+
output: str = typer.Option("zed_latest.png", "--output", "-o", help="PNG file to write."),
|
|
145
|
+
width: int = typer.Option(1280, help="Requested capture width."),
|
|
146
|
+
height: int = typer.Option(720, help="Requested capture height."),
|
|
147
|
+
fps: int = typer.Option(30, help="Requested capture frame rate."),
|
|
148
|
+
duration: float = typer.Option(1.0, "--duration", "-d", help="Seconds to read frames before saving."),
|
|
149
|
+
):
|
|
150
|
+
"""Open a ZED camera briefly and save the latest RGB frame as a PNG."""
|
|
151
|
+
output_path = Path(output)
|
|
152
|
+
if duration <= 0:
|
|
153
|
+
msg = "Duration must be greater than zero."
|
|
154
|
+
raise typer.BadParameter(msg)
|
|
155
|
+
if output_path.suffix.lower() != ".png":
|
|
156
|
+
msg = "Output path must end in .png."
|
|
157
|
+
raise typer.BadParameter(msg)
|
|
158
|
+
|
|
159
|
+
serial = _resolve_serial(serial)
|
|
160
|
+
camera = _create_rgb_camera(serial, width, height, fps)
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
camera.open()
|
|
164
|
+
except Exception as exc:
|
|
165
|
+
msg = f"Could not start ZED camera {serial}: {exc}"
|
|
166
|
+
raise typer.BadParameter(msg) from exc
|
|
167
|
+
|
|
168
|
+
latest_frame = None
|
|
169
|
+
frames_read = 0
|
|
170
|
+
deadline = monotonic() + duration
|
|
171
|
+
try:
|
|
172
|
+
while latest_frame is None or monotonic() < deadline:
|
|
173
|
+
latest_frame = camera.poll_frame("viewer")
|
|
174
|
+
frames_read += 1
|
|
175
|
+
finally:
|
|
176
|
+
camera.close()
|
|
177
|
+
|
|
178
|
+
assert latest_frame is not None
|
|
179
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
180
|
+
image_bgr = latest_frame.camera.color.data[:, :, ::-1]
|
|
181
|
+
print(latest_frame.camera.color.intrinsics)
|
|
182
|
+
if not cv2.imwrite(str(output_path), image_bgr):
|
|
183
|
+
msg = f"Could not write PNG to {output_path}."
|
|
184
|
+
raise typer.BadParameter(msg)
|
|
185
|
+
|
|
186
|
+
typer.echo(f"Saved {output} from ZED {serial} after reading {frames_read} frame(s).")
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def main():
|
|
190
|
+
zed_app()
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
if __name__ == "__main__":
|
|
194
|
+
main()
|
rcs_zed/camera.py
ADDED
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
import copy
|
|
2
|
+
import logging
|
|
3
|
+
import threading
|
|
4
|
+
import typing
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from time import time
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
from rcs.camera.hw import CalibrationStrategy, DummyCalibrationStrategy, HardwareCamera
|
|
10
|
+
from rcs.camera.interface import BaseCameraSet, CameraFrame, DataFrame, Frame, IMUFrame
|
|
11
|
+
|
|
12
|
+
from rcs import common
|
|
13
|
+
|
|
14
|
+
try:
|
|
15
|
+
from pyzed import sl
|
|
16
|
+
except ImportError: # pragma: no cover - exercised via fake backend tests
|
|
17
|
+
sl = None # type: ignore[assignment]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass
|
|
21
|
+
class ZEDDeviceInfo:
|
|
22
|
+
serial: str
|
|
23
|
+
model: str
|
|
24
|
+
has_depth: bool = True
|
|
25
|
+
has_imu: bool = False
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class ZEDFrameBundle:
|
|
30
|
+
color: np.ndarray
|
|
31
|
+
timestamp: float
|
|
32
|
+
color_intrinsics: np.ndarray[tuple[typing.Literal[3], typing.Literal[4]], np.dtype[np.float64]]
|
|
33
|
+
right_color: np.ndarray | None = None
|
|
34
|
+
right_color_intrinsics: np.ndarray[tuple[typing.Literal[3], typing.Literal[4]], np.dtype[np.float64]] | None = None
|
|
35
|
+
depth: np.ndarray | None = None
|
|
36
|
+
accel: np.ndarray | None = None
|
|
37
|
+
gyro: np.ndarray | None = None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _intrinsics_matrix(fx: float, fy: float, cx: float, cy: float) -> np.ndarray:
|
|
41
|
+
return np.array(
|
|
42
|
+
[
|
|
43
|
+
[fx, 0, cx, 0],
|
|
44
|
+
[0, fy, cy, 0],
|
|
45
|
+
[0, 0, 1, 0],
|
|
46
|
+
],
|
|
47
|
+
dtype=np.float64,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class PyZEDCameraHandle:
|
|
52
|
+
def __init__(
|
|
53
|
+
self,
|
|
54
|
+
camera: typing.Any,
|
|
55
|
+
device_info: ZEDDeviceInfo,
|
|
56
|
+
color_intrinsics: np.ndarray,
|
|
57
|
+
right_color_intrinsics: np.ndarray | None = None,
|
|
58
|
+
):
|
|
59
|
+
self.camera = camera
|
|
60
|
+
self.device_info = device_info
|
|
61
|
+
self.color_intrinsics = color_intrinsics
|
|
62
|
+
self.right_color_intrinsics = right_color_intrinsics
|
|
63
|
+
self.runtime_parameters = sl.RuntimeParameters() # type: ignore[union-attr]
|
|
64
|
+
self.image_mat = sl.Mat() # type: ignore[union-attr]
|
|
65
|
+
self.right_image_mat = sl.Mat() # type: ignore[union-attr]
|
|
66
|
+
self.depth_mat = sl.Mat() # type: ignore[union-attr]
|
|
67
|
+
self.sensors_data = sl.SensorsData() # type: ignore[union-attr]
|
|
68
|
+
|
|
69
|
+
def _timestamp_seconds(self) -> float:
|
|
70
|
+
try:
|
|
71
|
+
timestamp = self.camera.get_timestamp(sl.TIME_REFERENCE.IMAGE) # type: ignore[union-attr]
|
|
72
|
+
if hasattr(timestamp, "get_nanoseconds"):
|
|
73
|
+
return float(timestamp.get_nanoseconds()) * 1e-9
|
|
74
|
+
if hasattr(timestamp, "get_microseconds"):
|
|
75
|
+
return float(timestamp.get_microseconds()) * 1e-6
|
|
76
|
+
if hasattr(timestamp, "get_milliseconds"):
|
|
77
|
+
return float(timestamp.get_milliseconds()) * 1e-3
|
|
78
|
+
except Exception:
|
|
79
|
+
pass
|
|
80
|
+
return time()
|
|
81
|
+
|
|
82
|
+
def grab_frame(self, include_right: bool = False) -> ZEDFrameBundle:
|
|
83
|
+
err = self.camera.grab(self.runtime_parameters)
|
|
84
|
+
if err != sl.ERROR_CODE.SUCCESS: # type: ignore[union-attr]
|
|
85
|
+
msg = f"Failed to grab ZED frame: {err}"
|
|
86
|
+
raise RuntimeError(msg)
|
|
87
|
+
|
|
88
|
+
self.camera.retrieve_image(self.image_mat, sl.VIEW.LEFT) # type: ignore[union-attr]
|
|
89
|
+
color_raw = np.array(self.image_mat.get_data(), copy=True)
|
|
90
|
+
if color_raw.ndim != 3:
|
|
91
|
+
msg = f"Unexpected ZED image shape {color_raw.shape}"
|
|
92
|
+
raise RuntimeError(msg)
|
|
93
|
+
color_rgb = color_raw[:, :, :3][:, :, ::-1] if color_raw.shape[2] == 4 else color_raw[:, :, ::-1]
|
|
94
|
+
|
|
95
|
+
right_color = None
|
|
96
|
+
if include_right:
|
|
97
|
+
self.camera.retrieve_image(self.right_image_mat, sl.VIEW.RIGHT) # type: ignore[union-attr]
|
|
98
|
+
right_raw = np.array(self.right_image_mat.get_data(), copy=True)
|
|
99
|
+
if right_raw.ndim != 3:
|
|
100
|
+
msg = f"Unexpected ZED right image shape {right_raw.shape}"
|
|
101
|
+
raise RuntimeError(msg)
|
|
102
|
+
right_color = right_raw[:, :, :3][:, :, ::-1] if right_raw.shape[2] == 4 else right_raw[:, :, ::-1]
|
|
103
|
+
|
|
104
|
+
depth = None
|
|
105
|
+
if self.device_info.has_depth:
|
|
106
|
+
self.camera.retrieve_measure(self.depth_mat, sl.MEASURE.DEPTH) # type: ignore[union-attr]
|
|
107
|
+
depth_raw = np.array(self.depth_mat.get_data(), copy=True)
|
|
108
|
+
if depth_raw.ndim > 2:
|
|
109
|
+
depth_raw = depth_raw[:, :, 0]
|
|
110
|
+
depth_m = np.nan_to_num(depth_raw, nan=0.0, posinf=0.0, neginf=0.0)
|
|
111
|
+
depth_m = np.clip(depth_m, a_min=0.0, a_max=np.iinfo(np.uint16).max / BaseCameraSet.DEPTH_SCALE)
|
|
112
|
+
depth = (depth_m * BaseCameraSet.DEPTH_SCALE).astype(np.uint16)
|
|
113
|
+
|
|
114
|
+
accel = None
|
|
115
|
+
gyro = None
|
|
116
|
+
if self.device_info.has_imu:
|
|
117
|
+
sensor_err = self.camera.get_sensors_data(self.sensors_data, sl.TIME_REFERENCE.IMAGE) # type: ignore[union-attr]
|
|
118
|
+
if sensor_err == sl.ERROR_CODE.SUCCESS: # type: ignore[union-attr]
|
|
119
|
+
imu_data = self.sensors_data.get_imu_data()
|
|
120
|
+
if hasattr(imu_data, "get_linear_acceleration"):
|
|
121
|
+
accel = np.array(imu_data.get_linear_acceleration(), dtype=np.float64)
|
|
122
|
+
if hasattr(imu_data, "get_angular_velocity"):
|
|
123
|
+
gyro = np.array(imu_data.get_angular_velocity(), dtype=np.float64)
|
|
124
|
+
|
|
125
|
+
return ZEDFrameBundle(
|
|
126
|
+
color=color_rgb,
|
|
127
|
+
right_color=right_color,
|
|
128
|
+
right_color_intrinsics=self.right_color_intrinsics if right_color is not None else None,
|
|
129
|
+
depth=depth,
|
|
130
|
+
accel=accel,
|
|
131
|
+
gyro=gyro,
|
|
132
|
+
timestamp=self._timestamp_seconds(),
|
|
133
|
+
color_intrinsics=self.color_intrinsics,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
def close(self):
|
|
137
|
+
self.camera.close()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
class ZEDCameraSet(HardwareCamera):
|
|
141
|
+
CALIBRATION_FRAME_SIZE = 30
|
|
142
|
+
|
|
143
|
+
@staticmethod
|
|
144
|
+
def _require_sdk():
|
|
145
|
+
if sl is None:
|
|
146
|
+
msg = (
|
|
147
|
+
"The ZED SDK Python bindings are not available. Install the ZED SDK and ensure "
|
|
148
|
+
"`import pyzed.sl as sl` works on this machine."
|
|
149
|
+
)
|
|
150
|
+
raise RuntimeError(msg)
|
|
151
|
+
|
|
152
|
+
@staticmethod
|
|
153
|
+
def _device_has_imu(device: typing.Any) -> bool:
|
|
154
|
+
for attr in ("sensors_configuration", "sensors_conf"):
|
|
155
|
+
sensors_conf = getattr(device, attr, None)
|
|
156
|
+
if sensors_conf is None:
|
|
157
|
+
continue
|
|
158
|
+
camera_imu = getattr(sensors_conf, "camera_imu", None)
|
|
159
|
+
if camera_imu is None:
|
|
160
|
+
continue
|
|
161
|
+
if hasattr(camera_imu, "available"):
|
|
162
|
+
return bool(camera_imu.available)
|
|
163
|
+
return True
|
|
164
|
+
return False
|
|
165
|
+
|
|
166
|
+
@staticmethod
|
|
167
|
+
def _model_to_string(model: typing.Any) -> str:
|
|
168
|
+
if hasattr(model, "name"):
|
|
169
|
+
return str(model.name)
|
|
170
|
+
return str(model)
|
|
171
|
+
|
|
172
|
+
@staticmethod
|
|
173
|
+
def _map_resolution(width: int, height: int):
|
|
174
|
+
ZEDCameraSet._require_sdk()
|
|
175
|
+
assert sl is not None
|
|
176
|
+
mapping = {
|
|
177
|
+
(2208, 1242): sl.RESOLUTION.HD2K,
|
|
178
|
+
(1920, 1080): sl.RESOLUTION.HD1080,
|
|
179
|
+
(1280, 720): sl.RESOLUTION.HD720,
|
|
180
|
+
(672, 376): sl.RESOLUTION.VGA,
|
|
181
|
+
}
|
|
182
|
+
if (width, height) not in mapping:
|
|
183
|
+
msg = f"Unsupported ZED resolution {width}x{height}. Use one of: {sorted(mapping)}"
|
|
184
|
+
raise ValueError(msg)
|
|
185
|
+
return mapping[(width, height)]
|
|
186
|
+
|
|
187
|
+
@classmethod
|
|
188
|
+
def enumerate_connected_devices(cls) -> dict[str, ZEDDeviceInfo]:
|
|
189
|
+
cls._require_sdk()
|
|
190
|
+
assert sl is not None
|
|
191
|
+
devices: dict[str, ZEDDeviceInfo] = {}
|
|
192
|
+
for device in sl.Camera.get_device_list():
|
|
193
|
+
serial = str(device.serial_number)
|
|
194
|
+
devices[serial] = ZEDDeviceInfo(
|
|
195
|
+
serial=serial,
|
|
196
|
+
model=cls._model_to_string(device.camera_model),
|
|
197
|
+
has_depth=True,
|
|
198
|
+
has_imu=cls._device_has_imu(device),
|
|
199
|
+
)
|
|
200
|
+
return devices
|
|
201
|
+
|
|
202
|
+
@classmethod
|
|
203
|
+
def open_camera(
|
|
204
|
+
cls,
|
|
205
|
+
config: common.BaseCameraConfig,
|
|
206
|
+
*,
|
|
207
|
+
enable_depth: bool,
|
|
208
|
+
enable_imu: bool,
|
|
209
|
+
include_right: bool,
|
|
210
|
+
) -> PyZEDCameraHandle:
|
|
211
|
+
cls._require_sdk()
|
|
212
|
+
assert sl is not None
|
|
213
|
+
|
|
214
|
+
init = sl.InitParameters()
|
|
215
|
+
init.camera_resolution = cls._map_resolution(config.resolution_width, config.resolution_height)
|
|
216
|
+
init.camera_fps = config.frame_rate
|
|
217
|
+
init.coordinate_units = sl.UNIT.METER
|
|
218
|
+
init.depth_mode = sl.DEPTH_MODE.NONE if not enable_depth else sl.DEPTH_MODE.QUALITY
|
|
219
|
+
init.set_from_serial_number(int(config.identifier))
|
|
220
|
+
|
|
221
|
+
camera = sl.Camera()
|
|
222
|
+
err = camera.open(init)
|
|
223
|
+
if err != sl.ERROR_CODE.SUCCESS:
|
|
224
|
+
msg = f"Could not open ZED camera {config.identifier}: {err}"
|
|
225
|
+
raise RuntimeError(msg)
|
|
226
|
+
|
|
227
|
+
information = camera.get_camera_information()
|
|
228
|
+
calibration = information.camera_configuration.calibration_parameters
|
|
229
|
+
left_cam = calibration.left_cam
|
|
230
|
+
right_cam = calibration.right_cam
|
|
231
|
+
info = ZEDDeviceInfo(
|
|
232
|
+
serial=str(config.identifier),
|
|
233
|
+
model=cls._model_to_string(information.camera_model),
|
|
234
|
+
has_depth=enable_depth,
|
|
235
|
+
has_imu=enable_imu and cls._device_has_imu(information),
|
|
236
|
+
)
|
|
237
|
+
intrinsics = _intrinsics_matrix(left_cam.fx, left_cam.fy, left_cam.cx, left_cam.cy)
|
|
238
|
+
right_intrinsics = None
|
|
239
|
+
if include_right:
|
|
240
|
+
right_intrinsics = _intrinsics_matrix(right_cam.fx, right_cam.fy, right_cam.cx, right_cam.cy)
|
|
241
|
+
return PyZEDCameraHandle(
|
|
242
|
+
camera=camera,
|
|
243
|
+
device_info=info,
|
|
244
|
+
color_intrinsics=intrinsics,
|
|
245
|
+
right_color_intrinsics=right_intrinsics,
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
def __init__(
|
|
249
|
+
self,
|
|
250
|
+
cameras: dict[str, common.BaseCameraConfig],
|
|
251
|
+
calibration_strategy: dict[str, CalibrationStrategy] | None = None,
|
|
252
|
+
enable_depth: bool = True,
|
|
253
|
+
enable_imu: bool = True,
|
|
254
|
+
include_right: bool = False,
|
|
255
|
+
) -> None:
|
|
256
|
+
self.cameras = cameras
|
|
257
|
+
if calibration_strategy is None:
|
|
258
|
+
calibration_strategy = {camera_name: DummyCalibrationStrategy() for camera_name in cameras}
|
|
259
|
+
self.calibration_strategy = calibration_strategy
|
|
260
|
+
self.enable_depth = enable_depth
|
|
261
|
+
self.enable_imu = enable_imu
|
|
262
|
+
self.include_right = include_right
|
|
263
|
+
self._logger = logging.getLogger(__name__)
|
|
264
|
+
self._camera_names = self._logical_camera_names()
|
|
265
|
+
self._available_devices: dict[str, ZEDDeviceInfo] = {}
|
|
266
|
+
self._enabled_devices: dict[str, PyZEDCameraHandle] = {}
|
|
267
|
+
self._frame_buffer_lock: dict[str, threading.Lock] = {}
|
|
268
|
+
self._frame_buffer: dict[str, list[Frame]] = {}
|
|
269
|
+
self._pending_right_bundles: dict[str, ZEDFrameBundle] = {}
|
|
270
|
+
|
|
271
|
+
assert (
|
|
272
|
+
len({camera.resolution_width for camera in self.cameras.values()}) == 1
|
|
273
|
+
and len({camera.resolution_height for camera in self.cameras.values()}) == 1
|
|
274
|
+
and len({camera.frame_rate for camera in self.cameras.values()}) == 1
|
|
275
|
+
), "All cameras must have the same resolution and frame rate."
|
|
276
|
+
|
|
277
|
+
@property
|
|
278
|
+
def camera_names(self) -> list[str]:
|
|
279
|
+
return self._camera_names
|
|
280
|
+
|
|
281
|
+
def config(self, camera_name) -> common.BaseCameraConfig:
|
|
282
|
+
return self.cameras[self._physical_camera_name(camera_name)]
|
|
283
|
+
|
|
284
|
+
@staticmethod
|
|
285
|
+
def _right_camera_name(camera_name: str) -> str:
|
|
286
|
+
return f"{camera_name}_right"
|
|
287
|
+
|
|
288
|
+
def _physical_camera_name(self, camera_name: str) -> str:
|
|
289
|
+
if camera_name in self.cameras:
|
|
290
|
+
return camera_name
|
|
291
|
+
if self.include_right and camera_name.endswith("_right"):
|
|
292
|
+
physical_name = camera_name.removesuffix("_right")
|
|
293
|
+
if physical_name in self.cameras:
|
|
294
|
+
return physical_name
|
|
295
|
+
msg = f"Camera {camera_name} not found in the enabled devices"
|
|
296
|
+
raise AssertionError(msg)
|
|
297
|
+
|
|
298
|
+
def _is_right_camera_name(self, camera_name: str) -> bool:
|
|
299
|
+
return camera_name not in self.cameras and self._physical_camera_name(camera_name) != camera_name
|
|
300
|
+
|
|
301
|
+
def _logical_camera_names(self) -> list[str]:
|
|
302
|
+
names = list(self.cameras.keys())
|
|
303
|
+
if self.include_right:
|
|
304
|
+
names.extend(self._right_camera_name(camera_name) for camera_name in self.cameras)
|
|
305
|
+
return names
|
|
306
|
+
|
|
307
|
+
def _frame_from_bundle(self, camera_name: str, bundle: ZEDFrameBundle) -> Frame:
|
|
308
|
+
physical_name = self._physical_camera_name(camera_name)
|
|
309
|
+
extrinsics = self.calibration_strategy[physical_name].get_extrinsics()
|
|
310
|
+
is_right = self._is_right_camera_name(camera_name)
|
|
311
|
+
color_data = bundle.right_color if is_right else bundle.color
|
|
312
|
+
color_intrinsics = bundle.right_color_intrinsics if is_right else bundle.color_intrinsics
|
|
313
|
+
if color_data is None or color_intrinsics is None:
|
|
314
|
+
msg = f"Missing {'right' if is_right else 'left'} image data for {camera_name}"
|
|
315
|
+
raise RuntimeError(msg)
|
|
316
|
+
|
|
317
|
+
color = DataFrame(
|
|
318
|
+
data=color_data,
|
|
319
|
+
timestamp=bundle.timestamp,
|
|
320
|
+
intrinsics=color_intrinsics,
|
|
321
|
+
extrinsics=extrinsics,
|
|
322
|
+
)
|
|
323
|
+
depth = None
|
|
324
|
+
if not is_right and bundle.depth is not None:
|
|
325
|
+
depth = DataFrame(
|
|
326
|
+
data=bundle.depth,
|
|
327
|
+
timestamp=bundle.timestamp,
|
|
328
|
+
intrinsics=bundle.color_intrinsics,
|
|
329
|
+
extrinsics=extrinsics,
|
|
330
|
+
)
|
|
331
|
+
|
|
332
|
+
accel = DataFrame(data=bundle.accel, timestamp=bundle.timestamp) if bundle.accel is not None else None
|
|
333
|
+
gyro = DataFrame(data=bundle.gyro, timestamp=bundle.timestamp) if bundle.gyro is not None else None
|
|
334
|
+
return Frame(
|
|
335
|
+
camera=CameraFrame(color=color, depth=depth),
|
|
336
|
+
imu=IMUFrame(accel=accel, gyro=gyro) if (accel is not None or gyro is not None) else None,
|
|
337
|
+
avg_timestamp=bundle.timestamp,
|
|
338
|
+
)
|
|
339
|
+
|
|
340
|
+
def update_available_devices(self):
|
|
341
|
+
self._available_devices = self.enumerate_connected_devices()
|
|
342
|
+
|
|
343
|
+
def open(self):
|
|
344
|
+
self.update_available_devices()
|
|
345
|
+
self._enabled_devices = {}
|
|
346
|
+
self._frame_buffer = {}
|
|
347
|
+
self._frame_buffer_lock = {}
|
|
348
|
+
self._pending_right_bundles = {}
|
|
349
|
+
|
|
350
|
+
for camera_name, camera_cfg in self.cameras.items():
|
|
351
|
+
if camera_cfg.identifier not in self._available_devices:
|
|
352
|
+
msg = f"ZED device {camera_name} with serial {camera_cfg.identifier} not found."
|
|
353
|
+
raise RuntimeError(msg)
|
|
354
|
+
|
|
355
|
+
device_info = self._available_devices[camera_cfg.identifier]
|
|
356
|
+
opened = self.open_camera(
|
|
357
|
+
camera_cfg,
|
|
358
|
+
enable_depth=self.enable_depth and device_info.has_depth,
|
|
359
|
+
enable_imu=self.enable_imu and device_info.has_imu,
|
|
360
|
+
include_right=self.include_right,
|
|
361
|
+
)
|
|
362
|
+
self._enabled_devices[camera_name] = opened
|
|
363
|
+
self._frame_buffer[camera_name] = []
|
|
364
|
+
self._frame_buffer_lock[camera_name] = threading.Lock()
|
|
365
|
+
|
|
366
|
+
def poll_frame(self, camera_name: str) -> Frame:
|
|
367
|
+
assert camera_name in self.camera_names, f"Camera {camera_name} not found in the enabled devices"
|
|
368
|
+
physical_name = self._physical_camera_name(camera_name)
|
|
369
|
+
device = self._enabled_devices[physical_name]
|
|
370
|
+
|
|
371
|
+
if self._is_right_camera_name(camera_name):
|
|
372
|
+
bundle = self._pending_right_bundles.pop(physical_name, None)
|
|
373
|
+
if bundle is None:
|
|
374
|
+
bundle = device.grab_frame(include_right=True)
|
|
375
|
+
else:
|
|
376
|
+
bundle = device.grab_frame(include_right=self.include_right)
|
|
377
|
+
if self.include_right and bundle.right_color is not None:
|
|
378
|
+
self._pending_right_bundles[physical_name] = bundle
|
|
379
|
+
else:
|
|
380
|
+
self._pending_right_bundles.pop(physical_name, None)
|
|
381
|
+
|
|
382
|
+
frame = self._frame_from_bundle(camera_name, bundle)
|
|
383
|
+
|
|
384
|
+
if not self._is_right_camera_name(camera_name):
|
|
385
|
+
with self._frame_buffer_lock[physical_name]:
|
|
386
|
+
if len(self._frame_buffer[physical_name]) >= self.CALIBRATION_FRAME_SIZE:
|
|
387
|
+
self._frame_buffer[physical_name].pop(0)
|
|
388
|
+
self._frame_buffer[physical_name].append(copy.deepcopy(frame))
|
|
389
|
+
return frame
|
|
390
|
+
|
|
391
|
+
def close(self):
|
|
392
|
+
for device in self._enabled_devices.values():
|
|
393
|
+
device.close()
|
|
394
|
+
self._enabled_devices = {}
|
|
395
|
+
self._pending_right_bundles = {}
|
|
396
|
+
|
|
397
|
+
def calibrate(self) -> bool:
|
|
398
|
+
for camera_name in self.cameras:
|
|
399
|
+
device = self._enabled_devices.get(camera_name)
|
|
400
|
+
if device is None:
|
|
401
|
+
msg = f"Camera {camera_name} must be opened before calibration."
|
|
402
|
+
raise RuntimeError(msg)
|
|
403
|
+
if not self.calibration_strategy[camera_name].calibrate(
|
|
404
|
+
intrinsics=device.color_intrinsics,
|
|
405
|
+
samples=self._frame_buffer[camera_name],
|
|
406
|
+
lock=self._frame_buffer_lock[camera_name],
|
|
407
|
+
):
|
|
408
|
+
self._logger.warning("Calibration of camera %s failed.", camera_name)
|
|
409
|
+
return False
|
|
410
|
+
self._logger.info("Calibration successful.")
|
|
411
|
+
return True
|
rcs_zed/utils.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from rcs.camera.hw import CalibrationStrategy
|
|
2
|
+
from rcs_zed.camera import ZEDCameraSet
|
|
3
|
+
|
|
4
|
+
from rcs import common
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def default_zed(
|
|
8
|
+
name2id: dict[str, str] | None,
|
|
9
|
+
calibration_strategy: dict[str, CalibrationStrategy] | None = None,
|
|
10
|
+
) -> ZEDCameraSet | None:
|
|
11
|
+
"""Create the default ZED camera set.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
name2id: Mapping from logical camera names to ZED serial numbers.
|
|
15
|
+
calibration_strategy: Optional calibration strategy for each logical
|
|
16
|
+
camera. When omitted, ``ZEDCameraSet`` uses
|
|
17
|
+
``DummyCalibrationStrategy``.
|
|
18
|
+
"""
|
|
19
|
+
if name2id is None:
|
|
20
|
+
return None
|
|
21
|
+
cameras = {
|
|
22
|
+
name: common.BaseCameraConfig(identifier=id, resolution_width=1280, resolution_height=720, frame_rate=30)
|
|
23
|
+
for name, id in name2id.items()
|
|
24
|
+
}
|
|
25
|
+
return ZEDCameraSet(cameras=cameras, calibration_strategy=calibration_strategy)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def default_zed_dummy_calibration(name2id: dict[str, str] | None) -> ZEDCameraSet | None:
|
|
29
|
+
"""Create the default ZED camera set with dummy calibration."""
|
|
30
|
+
return default_zed(name2id)
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: rcs_zed
|
|
3
|
+
Version: 0.7.3
|
|
4
|
+
Summary: RCS ZED camera module
|
|
5
|
+
Author-email: Tobias Juelg <tobias.juelg@utn.de>
|
|
6
|
+
Maintainer-email: Tobias Juelg <tobias.juelg@utn.de>
|
|
7
|
+
License-Expression: AGPL-3.0-or-later
|
|
8
|
+
Requires-Python: >=3.11
|
|
9
|
+
Description-Content-Type: text/markdown
|
|
10
|
+
Requires-Dist: rcs-core>=0.7.3
|
|
11
|
+
Requires-Dist: opencv-python~=4.10.0
|
|
12
|
+
Requires-Dist: typer~=0.9
|
|
13
|
+
|
|
14
|
+
# RCS ZED Extension
|
|
15
|
+
|
|
16
|
+
Support for Stereolabs ZED cameras in RCS.
|
|
17
|
+
|
|
18
|
+
This extension depends on [`rcs-core`](https://pypi.org/project/rcs-core/).
|
|
19
|
+
Documentation: <https://robotcontrolstack.org/extensions/rcs_zed>
|
|
20
|
+
|
|
21
|
+
## Installation
|
|
22
|
+
|
|
23
|
+
- You need a PC with an Nvidia GPU and CUDA 12.8 installed.
|
|
24
|
+
- Install the ZED SDK from [Stereolabs](https://www.stereolabs.com/en-fr/developers/release).
|
|
25
|
+
- Install the Python bindings from the [zed-python-api](https://github.com/stereolabs/zed-python-api).
|
|
26
|
+
|
|
27
|
+
Install from PyPI:
|
|
28
|
+
|
|
29
|
+
```shell
|
|
30
|
+
pip install rcs-zed
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Warning: plain `pip install rcs-zed` will install the published `rcs-core` dependency from PyPI.
|
|
34
|
+
|
|
35
|
+
Install from a local checkout:
|
|
36
|
+
|
|
37
|
+
```shell
|
|
38
|
+
pip install -ve .
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
If you want this extension to use your local RCS checkout instead of the published `rcs-core` package, first install the main package from the repository root:
|
|
42
|
+
|
|
43
|
+
```shell
|
|
44
|
+
pip install -ve . --no-build-isolation
|
|
45
|
+
pip install -ve extensions/rcs_zed
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Calibration
|
|
49
|
+
|
|
50
|
+
`default_zed(...)` is standalone and uses RCS's identity
|
|
51
|
+
`DummyCalibrationStrategy` by default. To use measured extrinsics, pass one
|
|
52
|
+
calibration strategy per logical camera:
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from rcs_zed.utils import default_zed
|
|
56
|
+
|
|
57
|
+
calibration = {
|
|
58
|
+
"wrist": my_wrist_calibration,
|
|
59
|
+
"scene": my_scene_calibration,
|
|
60
|
+
}
|
|
61
|
+
cameras = default_zed(
|
|
62
|
+
{"wrist": "12345678", "scene": "87654321"},
|
|
63
|
+
calibration_strategy=calibration,
|
|
64
|
+
)
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Each value must implement the `rcs.camera.hw.CalibrationStrategy` protocol.
|
|
68
|
+
This keeps ZED installation independent of other camera extensions and lets
|
|
69
|
+
applications choose the calibration method that matches their robot setup.
|
|
70
|
+
|
|
71
|
+
## CLI
|
|
72
|
+
|
|
73
|
+
```shell
|
|
74
|
+
python -m rcs_zed serials
|
|
75
|
+
python -m rcs_zed rgb-view
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## Permissions
|
|
79
|
+
|
|
80
|
+
```shell
|
|
81
|
+
sudo usermod -a -G video,plugdev $USER
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Connection Diagnostics
|
|
85
|
+
|
|
86
|
+
- Use `ZED_Diagnostic` first if something appears broken.
|
|
87
|
+
- See the Stereolabs USB troubleshooting guide: <https://support.stereolabs.com/hc/en-us/articles/207635225-How-to-fix-USB-3-0-bandwidth-and-connection-issues>
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
rcs_zed/__init__.py,sha256=zYiFHqR7JwbvdK9dvKrh-RTNfUqjHUwC4CTcFAPVYLc,22
|
|
2
|
+
rcs_zed/__main__.py,sha256=HKNRK9a7mB4Kz4a26i4FUanU6SvgUX2Yrrzlj2N3D64,6683
|
|
3
|
+
rcs_zed/camera.py,sha256=vk0vrXaw-5QgARWpSI4KpflDLv0IoQyI51vaa5Sg_Mw,16858
|
|
4
|
+
rcs_zed/utils.py,sha256=okg3d9zG2qPFuEiUa_soPJBEep3PdL_OhLe2vv3Uu0g,1069
|
|
5
|
+
rcs_zed-0.7.3.dist-info/METADATA,sha256=GBqNphMRwquVyAF0Q3oPORnQeB04tF2J1ZeXKoUxceA,2390
|
|
6
|
+
rcs_zed-0.7.3.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
7
|
+
rcs_zed-0.7.3.dist-info/top_level.txt,sha256=rB0upoLlwl3ScEYrtGOepN6GBzKZkDDHmAq9d8lIN0Q,8
|
|
8
|
+
rcs_zed-0.7.3.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
rcs_zed
|