pyorbbec 1.0.1.33__cp312-cp312-win_amd64.whl

pyorbbecsdk/examples/net_device.py

@@ -0,0 +1,177 @@
+# ******************************************************************************
+#  Copyright (c) 2024 Orbbec 3D Technology, Inc
+#  
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.  
+#  You may obtain a copy of the License at
+#  
+#      http:# www.apache.org/licenses/LICENSE-2.0
+#  
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# ******************************************************************************
+
+import platform
+import cv2
+import numpy as np
+import av
+import io
+import threading
+import time
+import pygame
+import os
+from pyorbbecsdk import (Pipeline, Context, Config, OBSensorType, OBFormat, OBError)
+from utils import frame_to_bgr_image
+
+ESC_KEY = 27
+# Gemini 335Le
+GEMINI_335LE_PRODUCT_ID = 0x080E
+# Gemini 435Le
+GEMINI_435LE_PRODUCT_ID = 0x0815
+
+def get_stream_profile(pipeline, sensor_type, width, height, fmt, fps):
+    profile_list = pipeline.get_stream_profile_list(sensor_type)
+    try:
+        profile = profile_list.get_video_stream_profile(width, height, fmt, fps)
+    except OBError:
+        profile = profile_list.get_default_video_stream_profile()
+    return profile
+
+def decode_h26x_frame(decoder, byte_data):
+    try:
+        packet = av.Packet(byte_data)
+        frames = decoder.decode(packet)
+        for frame in frames:
+            return frame.to_ndarray(format='bgr24')
+    except av.AVError as e:
+        print(f"Decoding error: {e}")
+    return None
+
+class FrameProcessor(threading.Thread):
+    def __init__(self, decoder, display_width, display_height):
+        super().__init__()
+        self.decoder = decoder
+        self.latest_frame = None
+        self.processed_frame = None
+        self.lock = threading.Lock()
+        self.running = True
+        self.daemon = True
+        self.display_width = display_width
+        self.display_height = display_height
+
+    def run(self):
+        while self.running:
+            with self.lock:
+                if self.latest_frame is not None:
+                    color_image = decode_h26x_frame(self.decoder, self.latest_frame)
+                    if color_image is not None:
+                        # Resize the image to 1080p
+                        resized_image = cv2.resize(color_image, (self.display_width, self.display_height))
+                        rgb_image = cv2.cvtColor(resized_image, cv2.COLOR_BGR2RGB)
+                        self.processed_frame = rgb_image
+                    self.latest_frame = None
+            time.sleep(0.001)
+
+    def update_frame(self, frame):
+        with self.lock:
+            self.latest_frame = frame
+
+    def get_processed_frame(self):
+        with self.lock:
+            return self.processed_frame
+
+    def stop(self):
+        self.running = False
+
+def main():
+    ctx = Context()
+    ip = input("Enter the IP address of the device (default: 192.168.1.10): ") or "192.168.1.10"
+    device = ctx.create_net_device(ip, 8090)
+    if device is None:
+        print("Failed to create net device")
+        return
+
+    config = Config()
+    pipeline = Pipeline(device)
+
+    device_info = device.get_device_info()
+    # Gemini 335Le, Gemini 435Le
+    SUPPORTED_PIDS = {GEMINI_435LE_PRODUCT_ID, GEMINI_335LE_PRODUCT_ID}
+    if device_info.get_pid() in SUPPORTED_PIDS:
+        # Set up 1280*800 capture
+        print("Current device is GEMINI 435Le or GEMINI 335Le, use OBFormat.MJPG")
+        color_profile = get_stream_profile(pipeline, OBSensorType.COLOR_SENSOR, 1280, 800, OBFormat.MJPG, 10)
+    else:
+        # Set up 4K capture
+        color_profile = get_stream_profile(pipeline, OBSensorType.COLOR_SENSOR, 3840, 2160, OBFormat.H264, 25)
+
+    config.enable_stream(color_profile)
+    pipeline.start(config)
+
+    # Choose the correct decoder based on the format
+    if color_profile.get_format() == OBFormat.H264:
+        color_codec_name = 'h264'
+    elif color_profile.get_format() == OBFormat.MJPG:
+        color_codec_name = 'mjpeg'
+    else:
+        color_codec_name = 'hevc'
+
+    try:
+        decoder = av.codec.CodecContext.create(color_codec_name, 'r')
+    except av.AVError as e:
+        print(f"Failed to create decoder for {color_codec_name}: {e}")
+        pipeline.stop()
+        return
+
+    # Set display resolution to 720p
+    display_width, display_height = 1280, 720
+    frame_processor = FrameProcessor(decoder, display_width, display_height)
+    frame_processor.start()
+
+    pygame.init()
+    screen = pygame.display.set_mode((display_width, display_height))
+    pygame.display.set_caption("4K Net Device Viewer (720p Display)")
+    clock = pygame.time.Clock()
+
+    running = True
+    try:
+        while running:
+            for event in pygame.event.get():
+                if event.type == pygame.QUIT:
+                    running = False
+                elif event.type == pygame.KEYDOWN:
+                    if event.key == pygame.K_ESCAPE:
+                        running = False
+
+            if not running:
+                break
+
+            frames = pipeline.wait_for_frames(100)
+            if frames:
+                color_frame = frames.get_color_frame()
+                if color_frame:
+                    byte_data = color_frame.get_data()
+                    if len(byte_data) > 0:
+                        frame_processor.update_frame(byte_data)
+
+            processed_frame = frame_processor.get_processed_frame()
+            if processed_frame is not None:
+                surf = pygame.surfarray.make_surface(processed_frame.swapaxes(0, 1))
+                screen.blit(surf, (0, 0))
+                pygame.display.flip()
+
+            clock.tick(30)  # Limit to 30 FPS
+
+    finally:
+        print("Stopping frame processor...")
+        frame_processor.stop()
+        print("Stopping pipeline...")
+        pipeline.stop()
+        print("Exiting the program...")
+        os._exit(0)
+
+if __name__ == "__main__":
+    main()

pyorbbecsdk/examples/playback.py

@@ -0,0 +1,277 @@
+# ******************************************************************************
+#  Copyright (c) 2024 Orbbec 3D Technology, Inc
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http:# www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# ******************************************************************************
+
+import cv2
+import numpy as np
+from pyorbbecsdk import *
+from utils import frame_to_bgr_image
+
+# cached frames for better visualization
+cached_frames = {
+    'color': None,
+    'depth': None,
+    'left_ir': None,
+    'right_ir': None,
+    'ir': None
+}
+
+def setup_camera(playback):
+    """Setup camera and stream configuration"""
+    pipeline = Pipeline(playback)
+    config = Config()
+    device = pipeline.get_device()
+
+    # Try to enable all possible sensors
+    video_sensors = [
+        OBSensorType.COLOR_SENSOR,
+        OBSensorType.DEPTH_SENSOR,
+        OBSensorType.IR_SENSOR,
+        OBSensorType.LEFT_IR_SENSOR,
+        OBSensorType.RIGHT_IR_SENSOR,
+        OBSensorType.ACCEL_SENSOR, 
+        OBSensorType.GYRO_SENSOR, 
+    ]
+    enabled_sensor_types = []  
+
+    sensor_list = device.get_sensor_list()
+    for sensor in range(len(sensor_list)):
+        try:
+            sensor_type = sensor_list[sensor].get_type()
+            if sensor_type in video_sensors:
+                config.enable_stream(sensor_type)
+                enabled_sensor_types.append(sensor_type)
+        except:
+            continue
+    return pipeline, config, enabled_sensor_types
+
+def process_color(frame):
+    """Process color image"""
+    frame = frame if frame else cached_frames['color']
+    cached_frames['color'] = frame
+    return frame_to_bgr_image(frame) if frame else None
+
+
+def process_depth(frame):
+    """Process depth image"""
+    frame = frame if frame else cached_frames['depth']
+    cached_frames['depth'] = frame
+    if not frame:
+        return None
+    try:
+        depth_data = np.frombuffer(frame.get_data(), dtype=np.uint16)
+        depth_data = depth_data.reshape(frame.get_height(), frame.get_width())
+        depth_image = cv2.normalize(depth_data, None, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)
+        return cv2.applyColorMap(depth_image, cv2.COLORMAP_JET)
+    except ValueError:
+        return None
+
+
+def process_ir(ir_frame, key):
+    """Process IR frame (left, right, or mono) with cache"""  
+    ir_frame = ir_frame if ir_frame else cached_frames[key]
+    cached_frames[key] = ir_frame
+    if ir_frame is None:
+        return None
+
+    ir_frame = ir_frame.as_video_frame()
+    ir_data = np.asanyarray(ir_frame.get_data())
+    width = ir_frame.get_width()
+    height = ir_frame.get_height()
+    ir_format = ir_frame.get_format()
+
+    if ir_format == OBFormat.Y8:
+        ir_data = np.resize(ir_data, (height, width, 1))
+        data_type = np.uint8
+        image_dtype = cv2.CV_8UC1
+        max_data = 255
+    elif ir_format == OBFormat.MJPG:
+        ir_data = cv2.imdecode(ir_data, cv2.IMREAD_UNCHANGED)
+        data_type = np.uint8
+        image_dtype = cv2.CV_8UC1
+        max_data = 255
+        if ir_data is None:
+            print("decode mjpeg failed")
+            return None
+        ir_data = np.resize(ir_data, (height, width, 1))
+    else:
+        ir_data = np.frombuffer(ir_data, dtype=np.uint16)
+        data_type = np.uint16
+        image_dtype = cv2.CV_16UC1
+        max_data = 255
+        ir_data = np.resize(ir_data, (height, width, 1))
+
+    cv2.normalize(ir_data, ir_data, 0, max_data, cv2.NORM_MINMAX, dtype=image_dtype)
+    ir_data = ir_data.astype(data_type)
+    return cv2.cvtColor(ir_data, cv2.COLOR_GRAY2RGB)
+
+def get_imu_text(frame, name):
+    """Format IMU data"""
+    if not frame:
+        return []
+    return [
+        f"{name} x: {frame.get_x():.2f}",
+        f"{name} y: {frame.get_y():.2f}",
+        f"{name} z: {frame.get_z():.2f}"
+    ]
+
+
+
+def create_display(frames, enabled_sensor_types, width=1280, height=720):
+    """Create display window with correct dynamic layout"""
+    sensor_type_to_name = {
+        OBSensorType.COLOR_SENSOR: 'color',
+        OBSensorType.DEPTH_SENSOR: 'depth',
+        OBSensorType.LEFT_IR_SENSOR: 'left_ir',
+        OBSensorType.RIGHT_IR_SENSOR: 'right_ir',
+        OBSensorType.IR_SENSOR: 'ir'
+    }
+    video_keys = []
+    for sensor_type in enabled_sensor_types:
+        if sensor_type in sensor_type_to_name:
+            video_keys.append(sensor_type_to_name[sensor_type])
+
+    video_frames = [frames.get(k) for k in video_keys]
+    has_imu = 'imu' in frames
+    num_videos = len(video_frames)
+
+    total_elements = num_videos + (1 if has_imu else 0)
+
+    if total_elements == 1:
+        grid_cols, grid_rows = 1, 1
+    elif total_elements <= 2:
+        grid_cols, grid_rows = 2, 1
+    elif total_elements <= 4:
+        grid_cols, grid_rows = 2, 2
+    elif total_elements <= 5:
+        grid_cols, grid_rows = 2, 3
+    else:
+        raise ValueError("Too many elements! Maximum supported is 5.")
+
+    cell_w = width // grid_cols
+    cell_h = height // grid_rows
+
+    display = np.zeros((cell_h * grid_rows, cell_w * grid_cols, 3), dtype=np.uint8)
+
+    for idx, frame in enumerate(video_frames):
+        row = idx // grid_cols
+        col = idx % grid_cols
+        x_start = col * cell_w
+        y_start = row * cell_h
+        if frame is not None:
+            if total_elements == 1:
+                resized = cv2.resize(frame, (width, height))
+                display = resized
+            else:
+                resized = cv2.resize(frame, (cell_w, cell_h))
+                display[y_start:y_start + cell_h, x_start:x_start + cell_w] = resized
+        else:
+            cv2.rectangle(display, (x_start, y_start), (x_start + cell_w, y_start + cell_h), (0, 0, 0), -1)
+
+    if has_imu:
+        imu_idx = num_videos
+        row = imu_idx // grid_cols
+        col = imu_idx % grid_cols
+        x_start = col * cell_w
+        y_start = row * cell_h
+        cv2.rectangle(display, (x_start, y_start), (x_start + cell_w, y_start + cell_h), (50, 50, 50), -1)
+
+        y_offset = y_start + 30
+        for data_type in ['accel', 'gyro']:
+            text_lines = get_imu_text(frames['imu'].get(data_type), data_type.title())
+            for i, line in enumerate(text_lines):
+                cv2.putText(display, line, (x_start + 10, y_offset + i * 20),
+                            cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1, cv2.LINE_AA)
+            y_offset += 100
+
+    return display
+
+
+
+def main():
+    # Window settings
+    WINDOW_NAME = "MultiStream Playback(.bag) Viewer"
+    file_path = input("Enter output filename (.bag) and press Enter to start playbacking: ")
+
+    DISPLAY_WIDTH = 1280
+    DISPLAY_HEIGHT = 720
+    # initialize playback
+    playback  = PlaybackDevice(file_path)
+    # Initialize camera
+    pipeline, config, enabled_sensor_types = setup_camera(playback)
+    device = pipeline.get_device()
+    def on_status_change(status):
+        print(f"[Callback] status changed: {status}")
+        if status == PlaybackStatus.Stopped:
+            pipeline.stop()
+            pipeline.start(config)
+    playback.set_playback_status_change_callback(on_status_change)
+
+    pipeline.start(config)
+
+    cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
+    cv2.resizeWindow(WINDOW_NAME, DISPLAY_WIDTH, DISPLAY_HEIGHT)
+    processed_frames = {}
+    while True:
+        # Get all frames
+        frames = pipeline.wait_for_frames(100)
+        if not frames:
+            continue
+
+        # Process color image        
+        color_frame = frames.get_frame(OBFrameType.COLOR_FRAME)
+        if color_frame:
+            processed_frames['color'] = process_color(color_frame.as_video_frame())
+        # Process depth image
+        depth_frame = frames.get_frame(OBFrameType.DEPTH_FRAME)
+        if depth_frame:
+            processed_frames['depth'] = process_depth(depth_frame.as_video_frame())       
+        # Process left IR
+        left_ir_frame = frames.get_frame(OBFrameType.LEFT_IR_FRAME)
+        processed_frames['left_ir'] = process_ir(left_ir_frame, 'left_ir')
+
+        # Process right IR
+        right_ir_frame = frames.get_frame(OBFrameType.RIGHT_IR_FRAME)
+        processed_frames['right_ir'] = process_ir(right_ir_frame, 'right_ir')
+
+        # Process mono IR
+        ir_frame = frames.get_ir_frame()
+        processed_frames['ir'] = process_ir(ir_frame, 'ir')
+
+        # Process IMU data
+        accel = frames.get_frame(OBFrameType.ACCEL_FRAME)
+        gyro = frames.get_frame(OBFrameType.GYRO_FRAME)
+        if accel and gyro:
+            processed_frames['imu'] = {
+                'accel': accel.as_accel_frame(),
+                'gyro': gyro.as_gyro_frame()
+            }
+
+        # create display
+        display = create_display(processed_frames, enabled_sensor_types, DISPLAY_WIDTH, DISPLAY_HEIGHT)
+        cv2.imshow(WINDOW_NAME, display)
+
+        # check exit key
+        key = cv2.waitKey(1) & 0xFF
+        if key in (ord('q'), 27):
+            break
+
+    pipeline.stop()
+    playback  = None 
+    cv2.destroyAllWindows()
+
+
+if __name__ == "__main__":
+    main()

pyorbbecsdk/examples/point_cloud.py

@@ -0,0 +1,90 @@
+# ******************************************************************************
+#  Copyright (c) 2024 Orbbec 3D Technology, Inc
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http:# www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# ******************************************************************************
+
+import os
+import numpy as np
+
+from pyorbbecsdk import *
+
+save_points_dir = os.path.join(os.getcwd(), "point_clouds")
+if not os.path.exists(save_points_dir):
+    os.mkdir(save_points_dir)
+
+def main():
+    pipeline = Pipeline()
+    config = Config()
+    
+    # Configure depth stream
+    depth_profile_list = pipeline.get_stream_profile_list(OBSensorType.DEPTH_SENSOR)
+    if depth_profile_list is None:
+        print("No proper depth profile, cannot generate point cloud")
+        return
+    depth_profile = depth_profile_list.get_default_video_stream_profile()
+    config.enable_stream(depth_profile)
+
+    has_color_sensor = False
+    try:
+        # Configure color stream if available
+        profile_list = pipeline.get_stream_profile_list(OBSensorType.COLOR_SENSOR)
+        if profile_list is not None:
+            color_profile = profile_list.get_default_video_stream_profile()
+            config.enable_stream(color_profile)
+            has_color_sensor = True
+    except OBError as e:
+        print(e)
+
+    pipeline.enable_frame_sync()
+    pipeline.start(config)
+    
+    #camera_param = pipeline.get_camera_param()
+    align_filter = AlignFilter(align_to_stream=OBStreamType.COLOR_STREAM)
+    point_cloud_filter = PointCloudFilter()
+    #point_cloud_filter.set_camera_param(camera_param)
+
+    while True:
+        frames = pipeline.wait_for_frames(100)
+        if frames is None:
+            continue
+        
+        depth_frame = frames.get_depth_frame()
+        if depth_frame is None:
+            continue
+        
+        color_frame = frames.get_color_frame()
+        if has_color_sensor and color_frame is None:
+            continue
+        
+        frame = align_filter.process(frames)
+        #scale = depth_frame.get_depth_scale()
+        #point_cloud_filter.set_position_data_scaled(scale)
+
+        point_format = OBFormat.RGB_POINT if has_color_sensor and color_frame is not None else OBFormat.POINT
+        point_cloud_filter.set_create_point_format(point_format)
+
+        point_cloud_frame = point_cloud_filter.process(frame)
+        if point_cloud_frame is None:
+            continue
+        #save point cloud
+        save_point_cloud_to_ply(os.path.join(save_points_dir, "point_cloud.ply"), point_cloud_frame)
+        #save mesh to point cloud
+        #save_point_cloud_to_ply(os.path.join(save_points_dir, "point_cloud.ply"), point_cloud_frame, False, True, 50)        
+        break
+    print("stop pipeline")
+    pipeline.stop()
+
+
+if __name__ == "__main__":
+    main()

pyorbbecsdk/examples/post_processing.py

@@ -0,0 +1,119 @@
+# ******************************************************************************
+#  Copyright (c) 2024 Orbbec 3D Technology, Inc
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http:# www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# ******************************************************************************
+import sys
+
+import cv2
+import numpy as np
+
+from pyorbbecsdk import *
+
+ESC_KEY = 27
+PRINT_INTERVAL = 1  # seconds
+MIN_DEPTH = 20  # 20mm
+MAX_DEPTH = 10000  # 10000mm
+
+def main():
+    pipeline = Pipeline()
+    config = Config()
+    try:
+        profile_list = pipeline.get_stream_profile_list(OBSensorType.DEPTH_SENSOR)
+        depth_profile = profile_list.get_default_video_stream_profile()
+        config.enable_stream(depth_profile)
+    except Exception as e:
+        print(e)
+        return
+
+    try:
+        pipeline.enable_frame_sync()
+    except Exception as e:
+        print(e)
+
+    try:
+        pipeline.start(config)
+    except Exception as e:
+        print(e)
+        return
+
+    device = pipeline.get_device()
+    assert device is not None
+    depth_sensor = device.get_sensor(OBSensorType.DEPTH_SENSOR)
+    assert depth_sensor is not None
+    filter_list = depth_sensor.get_recommended_filters()
+    assert filter_list is not None
+    # print filter list
+    for i in range(len(filter_list)):
+        post_filter = filter_list[i]
+        if post_filter:
+            print("filter name: ", post_filter.get_name())
+            print("filter is enabled: ", post_filter.is_enabled())
+
+    while True:
+        try:
+            frames = pipeline.wait_for_frames(100)
+            if not frames:
+                continue
+            depth_frame = frames.get_depth_frame()
+            if not depth_frame:
+                continue
+            for i in range(len(filter_list)):
+                post_filter = filter_list[i]
+                if post_filter and post_filter.is_enabled() and depth_frame:
+                    depth_data_size = depth_frame.get_data()
+                    if len(depth_data_size) < (depth_frame.get_width() * depth_frame.get_height() * 2):
+                        # print("depth data is not complete")
+                        continue
+
+                    new_depth_frame = post_filter.process(depth_frame)
+                    depth_frame = new_depth_frame.as_depth_frame()
+            # for Y16 format depth frame, print the distance of the center pixel every 30 frames
+            width = depth_frame.get_width()
+            height = depth_frame.get_height()
+            scale = depth_frame.get_depth_scale()
+            depth_format = depth_frame.get_format()
+            if depth_format != OBFormat.Y16:
+                print("depth format is not Y16")
+                continue
+            try:
+                depth_data = np.frombuffer(depth_frame.get_data(), dtype=np.uint16)
+                depth_data = depth_data.reshape((height, width))
+            except ValueError:
+                print("Failed to reshape depth data")
+                continue
+
+            depth_data = depth_data.astype(np.float32) * scale
+            depth_data = np.where((depth_data > MIN_DEPTH) & (depth_data < MAX_DEPTH), depth_data, 0)
+            depth_data = depth_data.astype(np.uint16)
+            if depth_frame.get_format() == OBFormat.Y16 and depth_frame.get_index() % 30 == 0:
+                # print the distance of the center pixel
+                center_y = int(height / 2)
+                center_x = int(width / 2)
+                center_distance = depth_data[center_y, center_x]
+                print("center distance: ", center_distance)
+            depth_image = cv2.normalize(depth_data, None, 0, 255, cv2.NORM_MINMAX, dtype=cv2.CV_8U)
+            depth_image = cv2.applyColorMap(depth_image, cv2.COLORMAP_JET)
+
+            cv2.imshow("Depth Viewer", depth_image)
+            key = cv2.waitKey(1)
+            if key == ord('q') or key == ESC_KEY:
+                break
+        except KeyboardInterrupt:
+            break
+    cv2.destroyAllWindows()
+    pipeline.stop()
+
+
+if __name__ == "__main__":
+    main()