airo-camera-toolkit 2025.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. airo_camera_toolkit-2025.4.0/PKG-INFO +24 -0
  2. airo_camera_toolkit-2025.4.0/README.md +111 -0
  3. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/__init__.py +0 -0
  4. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/calibration/__init__.py +0 -0
  5. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/calibration/collect_calibration_data.py +158 -0
  6. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/calibration/compute_calibration.py +373 -0
  7. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/calibration/fiducial_markers.py +299 -0
  8. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/calibration/hand_eye_calibration.py +114 -0
  9. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/__init__.py +0 -0
  10. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/camera_discovery.py +111 -0
  11. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/manual_test_hw.py +123 -0
  12. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/multiprocess/__init__.py +0 -0
  13. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/multiprocess/multiprocess_rerun_logger.py +113 -0
  14. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgb_camera.py +467 -0
  15. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/multiprocess/multiprocess_rgbd_camera.py +447 -0
  16. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/multiprocess/multiprocess_stereo_rgbd_camera.py +358 -0
  17. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/multiprocess/multiprocess_video_recorder.py +126 -0
  18. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/opencv_videocapture/__init__.py +0 -0
  19. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/opencv_videocapture/opencv_videocapture.py +115 -0
  20. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/realsense/__init__.py +0 -0
  21. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/realsense/realsense.py +187 -0
  22. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/realsense/realsense_scan_profiles.py +97 -0
  23. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/zed/__init__.py +0 -0
  24. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/zed/zed.py +320 -0
  25. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cameras/zed/zed2i.py +13 -0
  26. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/cli.py +55 -0
  27. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/image_transforms/__init__.py +11 -0
  28. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/image_transforms/composed_transform.py +37 -0
  29. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/image_transforms/image_transform.py +58 -0
  30. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/image_transforms/transforms/__init__.py +0 -0
  31. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/image_transforms/transforms/crop.py +57 -0
  32. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/image_transforms/transforms/resize.py +70 -0
  33. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/image_transforms/transforms/rotate90.py +68 -0
  34. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/interfaces.py +200 -0
  35. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/pinhole_operations/__init__.py +7 -0
  36. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/pinhole_operations/projection.py +28 -0
  37. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/pinhole_operations/triangulation.py +80 -0
  38. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/pinhole_operations/unprojection.py +161 -0
  39. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/point_clouds/__init__.py +0 -0
  40. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/point_clouds/conversions.py +52 -0
  41. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/point_clouds/operations.py +84 -0
  42. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/point_clouds/visualization.py +24 -0
  43. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/py.typed +0 -0
  44. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/utils/__init__.py +0 -0
  45. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/utils/annotation_tool.py +345 -0
  46. airo_camera_toolkit-2025.4.0/airo_camera_toolkit/utils/image_converter.py +122 -0
  47. airo_camera_toolkit-2025.4.0/airo_camera_toolkit.egg-info/PKG-INFO +24 -0
  48. airo_camera_toolkit-2025.4.0/airo_camera_toolkit.egg-info/SOURCES.txt +58 -0
  49. airo_camera_toolkit-2025.4.0/airo_camera_toolkit.egg-info/dependency_links.txt +1 -0
  50. airo_camera_toolkit-2025.4.0/airo_camera_toolkit.egg-info/entry_points.txt +2 -0
  51. airo_camera_toolkit-2025.4.0/airo_camera_toolkit.egg-info/requires.txt +14 -0
  52. airo_camera_toolkit-2025.4.0/airo_camera_toolkit.egg-info/top_level.txt +1 -0
  53. airo_camera_toolkit-2025.4.0/setup.cfg +4 -0
  54. airo_camera_toolkit-2025.4.0/setup.py +33 -0
  55. airo_camera_toolkit-2025.4.0/test/test_config.py +37 -0
  56. airo_camera_toolkit-2025.4.0/test/test_fiducial_markers.py +80 -0
  57. airo_camera_toolkit-2025.4.0/test/test_image_converter.py +46 -0
  58. airo_camera_toolkit-2025.4.0/test/test_image_transforms.py +98 -0
  59. airo_camera_toolkit-2025.4.0/test/test_pinhole_operations.py +73 -0
  60. airo_camera_toolkit-2025.4.0/test/test_point_clouds.py +49 -0
@@ -0,0 +1,24 @@
1
+ Metadata-Version: 2.4
2
+ Name: airo_camera_toolkit
3
+ Version: 2025.4.0
4
+ Summary: Interfaces and common functionality to work with RGB(D) cameras for robotic manipulation at the Ghent University AI and Robotics Lab
5
+ Author: Thomas Lips
6
+ Author-email: thomas.lips@ugent.be
7
+ Requires-Dist: numpy<2.0
8
+ Requires-Dist: opencv-contrib-python==4.8.1.78
9
+ Requires-Dist: opencv-python-headless==4.8.1.78
10
+ Requires-Dist: matplotlib
11
+ Requires-Dist: rerun-sdk>=0.11.0
12
+ Requires-Dist: click
13
+ Requires-Dist: open3d
14
+ Requires-Dist: loguru
15
+ Requires-Dist: airo-typing==2025.4.0
16
+ Requires-Dist: airo-spatial-algebra==2025.4.0
17
+ Requires-Dist: airo-dataset-tools==2025.4.0
18
+ Provides-Extra: hand-eye-calibration
19
+ Requires-Dist: airo-robots; extra == "hand-eye-calibration"
20
+ Dynamic: author
21
+ Dynamic: author-email
22
+ Dynamic: provides-extra
23
+ Dynamic: requires-dist
24
+ Dynamic: summary
@@ -0,0 +1,111 @@
1
+ # airo-camera-toolkit
2
+ This package contains code for working with RGB(D) cameras, images and point clouds.
3
+
4
+
5
+ Overview of the functionality and the structure:
6
+ ```cs
7
+ airo-camera-toolkit/
8
+ ├── calibration/ # hand-eye extrinsics calibration
9
+ ├── cameras/ # actual camera drivers
10
+ ├── image_transformations/ # reversible geometric 2D transforms
11
+ ├── pinhole_operations/ # 2D-3D operations
12
+ ├── point_clouds/ # conversions and operations
13
+ ├── utils/ # a.o. annotation tool and converter
14
+ ├── interfaces.py
15
+ └── cli.py
16
+ ```
17
+
18
+ ## Installation
19
+ The `airo_camera_toolkit` package can be installed with pip by running (from this directory):
20
+ ```
21
+ pip install .
22
+ ```
23
+ This will already allow you to use the hardare-independent functionality of this package, e.g. image conversion and projection.
24
+ Depending on the hardware you are using, you might need to complete additional installation.
25
+ Instructions can be found in the following files:
26
+ * [ZED Installation](airo_camera_toolkit/cameras/zed/installation.md)
27
+ * [RealSense Installation](airo_camera_toolkit/cameras/realsense/realsense_installation.md)
28
+
29
+ Additionally, to ensure you have `airo-robots` installed for the hand-eye calibration, install the extra dependencies:
30
+ ```
31
+ pip install .[hand-eye-calibration]
32
+ ```
33
+
34
+ ## Getting started with cameras
35
+ Camera can be accessed by instantiating the corresponding class:, e.g. for a ZED camera:
36
+ ```python
37
+ from airo_camera_toolkit.cameras.zed import Zed
38
+ from airo_camera_toolkit.utils import ImageConverter
39
+ import cv2
40
+
41
+ camera = Zed(Zed.RESOLUTION_720, fps=30)
42
+
43
+ while True:
44
+ image_rgb_float = camera.get_rgb_image()
45
+ image_bgr = ImageConverter.from_numpy_format(image_rgb_float).image_in_opencv_format
46
+ cv2.imshow("Image", image_bgr)
47
+ key = cv2.waitKey(10)
48
+ if key == ord('q'):
49
+ break
50
+ ```
51
+
52
+ ## Hand-eye calibration
53
+
54
+ Find the pose of a camera relative to a robot. In the [`calibration`](./airo_camera_toolkit/calibration/) folder, run this to get started:
55
+
56
+ ```shell
57
+ airo-camera-toolkit hand-eye-calibration --help
58
+ ```
59
+
60
+ See [calibration/README.md](./airo_camera_toolkit/calibration/README.md) for more details.
61
+
62
+
63
+
64
+ ## Utils
65
+
66
+ ### Image format conversion
67
+ Camera by default return images as numpy 32-bit float RGB images with values between 0 to 1 through `get_rgb_image()`.
68
+ This is most convenient for subsequent processing, e.g. with neural networks.
69
+ For higher performance, 8-bit unsigned integer RGB images are also accessible through `get_rgb_image_as_int()`.
70
+
71
+ However, when using OpenCV, you will need conversion to BGR format.
72
+ For this you can use the `ImageConverter` class:
73
+ ```python
74
+ from airo_camera_toolkit.utils import ImageConverter
75
+
76
+ image_rgb_int = camera.get_rgb_image_as_int()
77
+ image_bgr = ImageConverter.from_numpy_int_format(image_rgb_int).image_in_opencv_format
78
+ ```
79
+
80
+
81
+ ### Annotation tool
82
+
83
+ See [annotation_tool.md](./airo_camera_toolkit/annotation_tool.md) for usage instructions.
84
+
85
+
86
+ ## Pinhole Operations
87
+
88
+ 2D - 3D geometric operations using the pinhole model. See [readme](./airo_camera_toolkit/pinhole_operations/Readme.md) for more information.
89
+
90
+
91
+ ## Image Transforms
92
+
93
+ See the [README](./airo_camera_toolkit/image_transforms/README.md) in the `image_transforms` folder for more details.
94
+
95
+ ## Real-time visualisation
96
+ For realtime visualisation of robotics data we strongly encourage using [rerun.io](https://www.rerun.io/) instead of manually hacking something together with opencv/pyqt/... No wrappers are needed here, just pip install the SDK. An example notebook to get to know this tool and its potential can be found [here](notebooks/rerun_tutorial.ipynb).
97
+ See this [README](./docs/rerun.md) for more details.
98
+
99
+ ## Point clouds
100
+ See the tutorial notebook [here](notebooks/point_cloud_tutorial.ipynb) for an introduction.
101
+
102
+ ## Multiprocessing
103
+ Camera processing can be computationally expensive.
104
+ If this is a problem for your application, see [multiprocess/README.md](./airo_camera_toolkit/cameras/multiprocess/README.md).
105
+
106
+ ## References
107
+ For more background on cameras, in particular on the meaning of intrinsics, extrinics, distortion coefficients, pinhole (and other) camera models, see:
108
+ - Szeliski - Computer vision: Algorithms and Applications, available [here](https://szeliski.org/Book/)
109
+ - https://web.eecs.umich.edu/~justincj/teaching/eecs442/WI2021/schedule.html
110
+ - https://learnopencv.com/geometry-of-image-formation/ (extrinsics & intrinsics)
111
+ - http://www.cs.cmu.edu/~16385/s17/Slides/11.1_Camera_matrix.pdf (idem)
@@ -0,0 +1,158 @@
1
+ import datetime
2
+ import json
3
+ import os
4
+ import time
5
+ from typing import Optional, Tuple
6
+
7
+ import click
8
+ import cv2
9
+ from airo_camera_toolkit.calibration.fiducial_markers import detect_and_visualize_charuco_pose
10
+ from airo_camera_toolkit.cameras.camera_discovery import click_camera_options, discover_camera
11
+ from airo_camera_toolkit.interfaces import RGBCamera
12
+ from airo_camera_toolkit.utils.image_converter import ImageConverter
13
+ from airo_dataset_tools.data_parsers.camera_intrinsics import CameraIntrinsics
14
+ from airo_dataset_tools.data_parsers.pose import Pose
15
+ from airo_robots.manipulators.hardware.ur_rtde import URrtde
16
+ from airo_robots.manipulators.position_manipulator import PositionManipulator
17
+ from airo_typing import HomogeneousMatrixType, OpenCVIntImageType
18
+
19
+
20
+ def create_calibration_data_dir(calibration_dir: Optional[str] = None) -> str:
21
+ """Ensures that a calibration_dir exists and has an empty "data" subfolder where new calibration samples can be
22
+ stored.
23
+
24
+ Args:
25
+ calibration_dir: Directory to save the calibration data to. If None, a directory with the current timestamp
26
+ will be created in the current working directory.
27
+
28
+ Returns:
29
+ Path to the "data" subfolder of the calibration_dir.
30
+ """
31
+ if calibration_dir is None:
32
+ datetime_str = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
33
+ calibration_dir = os.path.join(os.getcwd(), f"calibration_{datetime_str}")
34
+
35
+ os.makedirs(calibration_dir, exist_ok=True)
36
+
37
+ data_dir = os.path.join(calibration_dir, "data")
38
+
39
+ # If data_dir already exists, check whether it is empty
40
+ if os.path.exists(data_dir) and len(os.listdir(data_dir)) != 0:
41
+ raise ValueError(f"The data subfolder of {calibration_dir} already exists and is not empty.")
42
+
43
+ os.makedirs(data_dir, exist_ok=True)
44
+ return data_dir
45
+
46
+
47
+ def save_calibration_sample(
48
+ sample_index: int, robot: URrtde, camera: RGBCamera, data_dir: str
49
+ ) -> Tuple[HomogeneousMatrixType, OpenCVIntImageType]:
50
+ """Collect a single calibration sample and save to the data_dir.
51
+ A calibration data sample consists of an image and a TCP pose.
52
+
53
+ Args:
54
+ sample_index: index of the sample used for the names of the saved files.
55
+ robot: The robot being used to collect the data.
56
+ camera: The camera being used to collect the data.
57
+ data_dir: The directory to save the sample to.
58
+
59
+ Returns:
60
+ If charuco board detection is succesful, the TCP pose and the image, else None.
61
+ """
62
+ # Stop freedrive so robot is completely still at moment of the image capture
63
+ robot.rtde_control.endTeachMode()
64
+
65
+ ROBOT_STOP_WAIT_TIME = 0.5
66
+ time.sleep(ROBOT_STOP_WAIT_TIME)
67
+
68
+ image_rgb = camera.get_rgb_image_as_int()
69
+ image_bgr = ImageConverter.from_numpy_int_format(image_rgb).image_in_opencv_format
70
+
71
+ tcp_pose = robot.get_tcp_pose()
72
+
73
+ suffix = f"{sample_index:04d}"
74
+ image_filename = f"image_{suffix}.png"
75
+ tcp_pose_filename = f"tcp_pose_{suffix}.json"
76
+ image_filepath = os.path.join(data_dir, image_filename)
77
+ tcp_pose_filepath = os.path.join(data_dir, tcp_pose_filename)
78
+
79
+ cv2.imwrite(image_filepath, image_bgr)
80
+
81
+ pose = Pose.from_homogeneous_matrix(tcp_pose)
82
+ with open(tcp_pose_filepath, "w") as f:
83
+ json.dump(pose.model_dump(), f, indent=4)
84
+
85
+ robot.rtde_control.teachMode()
86
+
87
+ return tcp_pose, image_bgr
88
+
89
+
90
+ def collect_calibration_data(robot: PositionManipulator, camera: RGBCamera, calibration_dir: Optional[str]) -> None:
91
+ """Collect calibration data samples for hand-eye calibration.
92
+
93
+ Args:
94
+ robot: the robot to use for collecting the data.
95
+ camera: the camera to use for collecting the data.
96
+ calibration_dir: directory to save the calibration data to, if None a directory will be created
97
+ """
98
+ from loguru import logger
99
+
100
+ data_dir = create_calibration_data_dir(calibration_dir)
101
+
102
+ logger.info(f"Saving calibration data to {data_dir}")
103
+ logger.info("Press S to save a sample, Q to quit.")
104
+
105
+ resolution = camera.resolution
106
+ intrinsics = camera.intrinsics_matrix()
107
+
108
+ # Saving the intrinsics
109
+ camera_intrinsics = CameraIntrinsics.from_matrix_and_resolution(intrinsics, resolution)
110
+ intrinsics_filepath = os.path.join(data_dir, "intrinsics.json")
111
+ with open(intrinsics_filepath, "w") as f:
112
+ json.dump(camera_intrinsics.model_dump(exclude_none=True), f, indent=4)
113
+
114
+ window_name = "Calibration data collection"
115
+ cv2.namedWindow(window_name, cv2.WINDOW_NORMAL)
116
+
117
+ # For now, the robot is assumed to be a UR robot with RTDE interface, as we make use of the teach mode functions.
118
+ robot.rtde_control.teachMode() # type: ignore
119
+ sample_index = 0
120
+
121
+ while True:
122
+ # Live visualization of board detection
123
+ image_rgb = camera.get_rgb_image_as_int()
124
+ image = ImageConverter.from_numpy_int_format(image_rgb).image_in_opencv_format
125
+ detect_and_visualize_charuco_pose(image, intrinsics)
126
+ cv2.imshow(window_name, image)
127
+
128
+ key = cv2.waitKey(1)
129
+ if key == ord("q"):
130
+ robot.rtde_control.endTeachMode() # type: ignore
131
+ break
132
+
133
+ if key == ord("s"):
134
+ save_calibration_sample(sample_index, robot, camera, data_dir) # type: ignore
135
+ sample_index += 1
136
+ logger.info(f"Saved {sample_index} sample(s).")
137
+
138
+
139
+ @click.command()
140
+ @click.option("--robot_ip", default="10.42.0.162", help="robot ip address")
141
+ @click.option("--calibration_dir", type=click.Path(exists=False), help="directory to save the calibration data to.")
142
+ @click_camera_options
143
+ def collect_calibration_data_with_ur(
144
+ robot_ip: str,
145
+ calibration_dir: Optional[str] = None,
146
+ camera_brand: Optional[str] = None,
147
+ camera_serial_number: Optional[str] = None,
148
+ ) -> None:
149
+ """Script to collect calibration data for hand-eye calibration with a UR robot."""
150
+ from airo_robots.manipulators.hardware.ur_rtde import URrtde
151
+
152
+ robot = URrtde(robot_ip, URrtde.UR3_CONFIG)
153
+ camera = discover_camera(camera_brand, camera_serial_number)
154
+ collect_calibration_data(robot, camera, calibration_dir)
155
+
156
+
157
+ if __name__ == "__main__":
158
+ collect_calibration_data_with_ur()
@@ -0,0 +1,373 @@
1
+ import datetime
2
+ import glob
3
+ import json
4
+ import os
5
+ from typing import List, Optional, Tuple
6
+
7
+ import click
8
+ import cv2
9
+ import numpy as np
10
+ from airo_camera_toolkit.calibration.fiducial_markers import (
11
+ AIRO_DEFAULT_ARUCO_DICT,
12
+ AIRO_DEFAULT_CHARUCO_BOARD,
13
+ ArucoDictType,
14
+ CharucoBoardType,
15
+ detect_charuco_board,
16
+ draw_frame_on_image,
17
+ )
18
+ from airo_dataset_tools.data_parsers.camera_intrinsics import CameraIntrinsics
19
+ from airo_dataset_tools.data_parsers.pose import Pose
20
+ from airo_spatial_algebra import SE3Container
21
+ from airo_typing import CameraIntrinsicsMatrixType, CameraResolutionType, HomogeneousMatrixType, OpenCVIntImageType
22
+ from loguru import logger
23
+
24
+ cv2_CALIBRATION_METHODS = {
25
+ "Tsai": cv2.CALIB_HAND_EYE_TSAI,
26
+ "Park": cv2.CALIB_HAND_EYE_PARK,
27
+ "Haraud": cv2.CALIB_HAND_EYE_HORAUD,
28
+ "Andreff": cv2.CALIB_HAND_EYE_ANDREFF,
29
+ "Daniilidis": cv2.CALIB_HAND_EYE_DANIILIDIS,
30
+ }
31
+
32
+
33
+ def compute_hand_eye_calibration_error(
34
+ tcp_poses_in_base: List[HomogeneousMatrixType],
35
+ board_poses_in_camera: List[HomogeneousMatrixType],
36
+ camera_pose: HomogeneousMatrixType,
37
+ ) -> float:
38
+ """Compute the error between the left and right side of the AX=XB equation to have an estimate of the error of the
39
+ calibration. In our experience, average error below 0.01 are pretty good.
40
+
41
+ Args:
42
+ tcp_poses_in_base: list of tcp poses in base frame
43
+ board_poses_in_camera: list of marker poses in camera frame
44
+ camera_pose: camera pose in base frame (eye-to-hand) or camera pose in tcp frame(eye-in-hand))
45
+ """
46
+ error = 0.0
47
+ for i in range(len(tcp_poses_in_base) - 1):
48
+ tcp_pose_in_base = tcp_poses_in_base[i]
49
+ board_pose_in_camera = board_poses_in_camera[i]
50
+
51
+ tcp_pose_in_base_2 = tcp_poses_in_base[i + 1]
52
+ board_pose_in_camera_2 = board_poses_in_camera[i + 1]
53
+
54
+ # cf https://docs.opencv.org/4.x/d9/d0c/group__calib3d.html#gaebfc1c9f7434196a374c382abf43439b
55
+ # for the AX=XB equation
56
+ left_side = tcp_pose_in_base @ camera_pose @ board_pose_in_camera
57
+ right_side = tcp_pose_in_base_2 @ camera_pose @ board_pose_in_camera_2
58
+ error += float(np.linalg.norm(left_side - right_side))
59
+ return error / (len(tcp_poses_in_base) - 1)
60
+
61
+
62
+ def eye_in_hand_pose_estimation(
63
+ tcp_poses_in_base: List[HomogeneousMatrixType],
64
+ board_poses_in_camera: List[HomogeneousMatrixType],
65
+ method: int = cv2.CALIB_HAND_EYE_ANDREFF,
66
+ ) -> Tuple[Optional[HomogeneousMatrixType], Optional[float]]:
67
+ """Wrapper around the opencv eye-in-hand extrinsics calibration function.
68
+
69
+ Args:
70
+ tcp_poses_in_base: list of tcp poses in base frame
71
+ board_poses_in_camera: list of marker poses in camera frame
72
+ method: one of the cv2.CALIB_HAND_EYE_* methods
73
+ """
74
+ tcp_orientations_as_rotvec_in_base = [
75
+ SE3Container.from_homogeneous_matrix(tcp_pose).orientation_as_rotation_vector for tcp_pose in tcp_poses_in_base
76
+ ]
77
+ tcp_positions_in_base = [
78
+ SE3Container.from_homogeneous_matrix(tcp_pose).translation for tcp_pose in tcp_poses_in_base
79
+ ]
80
+
81
+ marker_orientations_as_rotvec_in_camera = [
82
+ SE3Container.from_homogeneous_matrix(board_pose).orientation_as_rotation_vector
83
+ for board_pose in board_poses_in_camera
84
+ ]
85
+ marker_positions_in_camera = [
86
+ SE3Container.from_homogeneous_matrix(board_pose).translation for board_pose in board_poses_in_camera
87
+ ]
88
+
89
+ # When running with duplicated tcp poses, I've had this error:
90
+ # error: (-7:Iterations do not converge) Rotation normalization issue: determinant(R) is null in function 'normalizeRotation'
91
+ try:
92
+ camera_rotation_matrix, camera_translation = cv2.calibrateHandEye(
93
+ tcp_orientations_as_rotvec_in_base,
94
+ tcp_positions_in_base,
95
+ marker_orientations_as_rotvec_in_camera,
96
+ marker_positions_in_camera,
97
+ None,
98
+ None,
99
+ method,
100
+ )
101
+ except cv2.error:
102
+ return None, None
103
+
104
+ if camera_rotation_matrix is None or camera_translation is None:
105
+ return None, None
106
+
107
+ # We've noticed that the OpenCV output can contains NaNs, which crashes here.
108
+ try:
109
+ camera_pose_in_tcp_frame = SE3Container.from_rotation_matrix_and_translation(
110
+ camera_rotation_matrix, camera_translation
111
+ ).homogeneous_matrix
112
+ except ValueError:
113
+ return None, None
114
+
115
+ camera_pose_in_tcp_frame = SE3Container.from_rotation_matrix_and_translation(
116
+ camera_rotation_matrix, camera_translation
117
+ ).homogeneous_matrix
118
+
119
+ calibration_error = compute_hand_eye_calibration_error(
120
+ tcp_poses_in_base, board_poses_in_camera, camera_pose_in_tcp_frame
121
+ )
122
+ return camera_pose_in_tcp_frame, calibration_error
123
+
124
+
125
+ def eye_to_hand_pose_estimation(
126
+ tcp_poses_in_base: List[HomogeneousMatrixType],
127
+ board_poses_in_camera: List[HomogeneousMatrixType],
128
+ method: int = cv2.CALIB_HAND_EYE_ANDREFF,
129
+ ) -> Tuple[Optional[HomogeneousMatrixType], Optional[float]]:
130
+ """Wrapper around the opencv eye-to-hand extrinsics calibration function.
131
+
132
+ Args:
133
+ tcp_poses_in_base: list of tcp poses in base frame
134
+ board_poses_in_camera: list of marker poses in camera frame
135
+ method: one of the cv2.CALIB_HAND_EYE_* methods
136
+ """
137
+ # Invert the tcp_poses to make the AX=XB problem for eye_to_hand mode equivalent to the eye_in_hand mode.
138
+ # cf https://docs.opencv.org/4.5.4/d9/d0c/group__calib3d.html#gaebfc1c9f7434196a374c382abf43439b
139
+ # cf https://forum.opencv.org/t/eye-to-hand-calibration/5690/2
140
+ base_pose_in_tcp_frame = [np.linalg.inv(tcp_pose) for tcp_pose in tcp_poses_in_base]
141
+
142
+ camera_pose_in_base, calibration_error = eye_in_hand_pose_estimation(
143
+ base_pose_in_tcp_frame, board_poses_in_camera, method
144
+ )
145
+ return camera_pose_in_base, calibration_error
146
+
147
+
148
+ def compute_calibration(
149
+ board_poses_in_camera: List[HomogeneousMatrixType],
150
+ tcp_poses_in_base: List[HomogeneousMatrixType],
151
+ mode: str = "eye_in_hand",
152
+ method: int = cv2.CALIB_HAND_EYE_ANDREFF,
153
+ ) -> Tuple[Optional[HomogeneousMatrixType], Optional[float]]:
154
+ """Compute the calibration for a given mode and method.
155
+
156
+ Args:
157
+ board_poses_in_camera: list of marker poses in camera frame
158
+ tcp_poses_in_base: list of tcp poses in base frame
159
+ mode: one of "eye_in_hand" or "eye_to_hand"
160
+ method: one of the cv2.CALIB_HAND_EYE_* methods
161
+
162
+ Returns:
163
+ camera_pose: if successful, camera pose in base frame (eye-to-hand) or camera pose in tcp frame(eye-in-hand))
164
+ calibration_error: error of the calibration
165
+ """
166
+ if mode == "eye_in_hand":
167
+ # pose of camera in tcp frame
168
+ camera_pose, calibration_error = eye_in_hand_pose_estimation(tcp_poses_in_base, board_poses_in_camera, method)
169
+ elif mode == "eye_to_hand":
170
+ # pose of camera in base frame
171
+ camera_pose, calibration_error = eye_to_hand_pose_estimation(tcp_poses_in_base, board_poses_in_camera, method)
172
+ else:
173
+ raise ValueError(f"Unknown mode {mode}")
174
+
175
+ return camera_pose, calibration_error
176
+
177
+
178
+ def save_board_detections(
179
+ results_dir: str,
180
+ board_poses_in_camera: List[Optional[HomogeneousMatrixType]],
181
+ images: List[OpenCVIntImageType],
182
+ intrinsics: CameraIntrinsicsMatrixType,
183
+ ) -> None:
184
+ """Convenience function to that saves jpg images of with the board pose drawn on it.
185
+
186
+ Args:
187
+ results_dir: directory to save the results to, must exist
188
+ board_poses_in_camera: list of marker poses in camera frame, may contain None
189
+ images: list of images of the calibration board
190
+ intrinsics: camera intrinsics
191
+ """
192
+
193
+ board_detections_dir = os.path.join(results_dir, "board_detections")
194
+ os.makedirs(board_detections_dir)
195
+ for i, (board_pose, image) in enumerate(zip(board_poses_in_camera, images)):
196
+ image_annotated = image.copy()
197
+ if board_pose is None:
198
+ continue
199
+ draw_frame_on_image(image_annotated, board_pose, intrinsics)
200
+ detection_filepath = os.path.join(board_detections_dir, f"board_detection_{i:04d}.jpg")
201
+ cv2.imwrite(detection_filepath, image_annotated)
202
+
203
+
204
+ def draw_base_pose_on_image(
205
+ image: OpenCVIntImageType,
206
+ intrinsics: CameraIntrinsicsMatrixType,
207
+ camera_pose: Optional[HomogeneousMatrixType],
208
+ mode: str = "eye_in_hand",
209
+ tcp_pose: Optional[HomogeneousMatrixType] = None,
210
+ ) -> None:
211
+ """Draws the robot's base pose on an image, using the camera_pose resulting from the calibration.
212
+
213
+ Args:
214
+ image: image to draw on
215
+ camera_pose: camera pose in base frame (eye-to-hand) or camera pose in tcp frame(eye-in-hand))
216
+ intrinsics: camera intrinsics
217
+ mode: one of "eye_in_hand" or "eye_to_hand"
218
+ tcp_pose: tcp pose in base frame that corresponds to the image
219
+ """
220
+ if camera_pose is None:
221
+ return
222
+
223
+ if mode == "eye_to_hand":
224
+ X_B_C = camera_pose # Camera in base frame
225
+ X_C_B = np.linalg.inv(X_B_C)
226
+ if mode == "eye_in_hand":
227
+ if tcp_pose is None:
228
+ return # tcp pose is required to visualize base in eye_in_hand mode
229
+
230
+ X_TCP_C = camera_pose # Camera in TCP frame
231
+ X_B_TCP = tcp_pose
232
+ X_C_TCP = np.linalg.inv(X_TCP_C)
233
+ X_TCP_B = np.linalg.inv(X_B_TCP)
234
+ X_C_B = X_C_TCP @ X_TCP_B
235
+
236
+ base_pose_in_camera = X_C_B
237
+ draw_frame_on_image(image, base_pose_in_camera, intrinsics)
238
+
239
+
240
+ def compute_calibration_all_methods(
241
+ results_dir: str,
242
+ images: List[OpenCVIntImageType],
243
+ tcp_poses_in_base: List[HomogeneousMatrixType],
244
+ intrinsics: CameraIntrinsicsMatrixType,
245
+ mode: str = "eye_in_hand",
246
+ aruco_dict: ArucoDictType = AIRO_DEFAULT_ARUCO_DICT,
247
+ charuco_board: CharucoBoardType = AIRO_DEFAULT_CHARUCO_BOARD,
248
+ ) -> Tuple[dict, dict]:
249
+ """Computes the calibration solution for all methods available in OpenCV and saves the results to a directory.
250
+
251
+ Args:
252
+ results_dir: directory to save the results to, must exist
253
+ images: list of images of the calibration board
254
+ tcp_poses_in_base: list of tcp poses in base frame
255
+ intrinsics: camera intrinsics
256
+ mode: one of "eye_in_hand" or "eye_to_hand"
257
+ aruco_dict: aruco dictionary
258
+ charuco_board: charuco board
259
+
260
+ Returns:
261
+ calibration_result_poses: dictionary of the camera pose for each method
262
+ calibration_errors: dictionary of the calibration error for each method
263
+ """
264
+ calibration_errors_filepath = os.path.join(results_dir, "residual_errors.json")
265
+ calibration_errors = {}
266
+ calibration_result_poses = {}
267
+
268
+ board_poses_in_camera = [
269
+ detect_charuco_board(image, intrinsics, aruco_dict=aruco_dict, charuco_board=charuco_board) for image in images
270
+ ]
271
+
272
+ save_board_detections(results_dir, board_poses_in_camera, images, intrinsics)
273
+
274
+ # Removes poses where no board was detected
275
+ tcp_poses_in_base = [
276
+ tcp_poses_in_base[i] for i, board_pose in enumerate(board_poses_in_camera) if board_pose is not None
277
+ ]
278
+ board_poses_in_camera: List[HomogeneousMatrixType] = [ # type: ignore
279
+ board_pose for board_pose in board_poses_in_camera if board_pose is not None
280
+ ]
281
+ logger.info(f"Board poses were detected in {len(board_poses_in_camera)} of the calibration samples.")
282
+
283
+ for name, method in cv2_CALIBRATION_METHODS.items():
284
+ camera_pose, calibration_error = compute_calibration(board_poses_in_camera, tcp_poses_in_base, mode, method) # type: ignore
285
+ if calibration_error is None:
286
+ calibration_error = np.inf
287
+
288
+ logger.info(f"Residual error {name}: {calibration_error:.4f}")
289
+
290
+ calibration_errors[name] = calibration_error
291
+ calibration_result_poses[name] = camera_pose
292
+
293
+ with open(calibration_errors_filepath, "w") as f:
294
+ json.dump(calibration_errors, f, indent=4)
295
+
296
+ if camera_pose is None:
297
+ continue
298
+
299
+ # Save the camera pose
300
+ pose_path = os.path.join(results_dir, f"camera_pose_{name}.json")
301
+ pose_saveable = Pose.from_homogeneous_matrix(camera_pose)
302
+ with open(pose_path, "w") as f:
303
+ json.dump(pose_saveable.model_dump(), f, indent=4)
304
+
305
+ # Save an image with the pose drawn on it (use last image taken)
306
+ image = images[-1].copy()
307
+ draw_base_pose_on_image(image, intrinsics, camera_pose, mode, tcp_poses_in_base[-1])
308
+
309
+ # Write residual error on image
310
+ error_str = f"{name}: {calibration_error:.4f}"
311
+ cv2.putText(image, error_str, (10, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 255, 0), 2, cv2.LINE_AA)
312
+ cv2.imwrite(os.path.join(results_dir, f"base_pose_in_camera_{name}.jpg"), image)
313
+
314
+ return calibration_result_poses, calibration_errors
315
+
316
+
317
+ def load_calibration_data(
318
+ calibration_dir: str,
319
+ ) -> Tuple[List[OpenCVIntImageType], List[HomogeneousMatrixType], CameraIntrinsicsMatrixType, CameraResolutionType]:
320
+ """Function to load calibration samples and camera parameters from a "data" directory in a calibration_dir
321
+
322
+ Args:
323
+ calibration_dir: directory containing the "data" directory
324
+
325
+ Returns:
326
+ Calibration data and camera parameters
327
+ """
328
+ data_dir = os.path.join(calibration_dir, "data")
329
+
330
+ # Loading the intrinsics and resolution
331
+ intrinsics_path = os.path.join(data_dir, "intrinsics.json")
332
+ with open(intrinsics_path, "r") as f:
333
+ camera_intrinsics = CameraIntrinsics.model_validate_json(f.read())
334
+
335
+ resolution = camera_intrinsics.image_resolution.as_tuple()
336
+ intrinsics = camera_intrinsics.as_matrix()
337
+
338
+ image_paths = sorted(glob.glob(os.path.join(data_dir, "image_*.png")))
339
+ pose_paths = sorted(glob.glob(os.path.join(data_dir, "tcp_pose_*.json")))
340
+
341
+ images = [cv2.imread(image_path) for image_path in image_paths]
342
+ tcp_poses = []
343
+ for filepath in pose_paths:
344
+ with open(filepath, "r") as f:
345
+ pose = Pose.model_validate_json(f.read())
346
+ tcp_poses.append(pose.as_homogeneous_matrix())
347
+
348
+ return images, tcp_poses, intrinsics, resolution
349
+
350
+
351
+ @click.command()
352
+ @click.argument(
353
+ "calibration_dir",
354
+ type=click.Path(exists=True),
355
+ )
356
+ @click.option("--mode", default="eye_in_hand", help="eye_in_hand or eye_to_hand")
357
+ def compute_calibration_from_saved_data(calibration_dir: str, mode: str = "eye_in_hand") -> None:
358
+ """Runs all OpenCV calibration methods on the data saved in a calibration directory."""
359
+ images, tcp_poses, intrinsics, _ = load_calibration_data(calibration_dir)
360
+
361
+ results_dir = os.path.join(calibration_dir, "results")
362
+ if os.path.exists(results_dir):
363
+ datetime_str = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
364
+ results_dir = os.path.join(calibration_dir, f"results_{datetime_str}")
365
+ os.makedirs(results_dir)
366
+
367
+ logger.info(f"Saving calibration results to {results_dir}")
368
+
369
+ compute_calibration_all_methods(results_dir, images, tcp_poses, intrinsics, mode)
370
+
371
+
372
+ if __name__ == "__main__":
373
+ compute_calibration_from_saved_data()