supervisely 6.73.321__tar.gz → 6.73.322__tar.gz

{supervisely-6.73.321/supervisely.egg-info → supervisely-6.73.322}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.321
+Version: 6.73.322
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.321 → supervisely-6.73.322}/supervisely/convert/base_converter.py

@@ -65,6 +65,7 @@ class AvailablePointcloudEpisodesConverters:
     SLY = "supervisely"
     BAG = "rosbag"
     LYFT = "lyft"
+    KITTI360 = "kitti360"
 
 
 class AvailableVolumeConverters:

{supervisely-6.73.321 → supervisely-6.73.322}/supervisely/convert/pointcloud_episodes/__init__.py

@@ -7,3 +7,4 @@ from supervisely.convert.pointcloud_episodes.lyft.lyft_converter import LyftEpis
 from supervisely.convert.pointcloud_episodes.nuscenes_conv.nuscenes_converter import (
     NuscenesEpisodesConverter,
 )
+from supervisely.convert.pointcloud_episodes.kitti_360.kitti_360_converter import KITTI360Converter

supervisely-6.73.322/supervisely/convert/pointcloud_episodes/kitti_360/kitti_360_converter.py

@@ -0,0 +1,242 @@
+import os
+from pathlib import Path
+from typing import Optional, List
+from supervisely import PointcloudEpisodeAnnotation, ProjectMeta, is_development, logger, ObjClass, ObjClassCollection
+from supervisely.geometry.cuboid_3d import Cuboid3d
+from supervisely.api.api import Api, ApiField
+from supervisely.convert.base_converter import AvailablePointcloudEpisodesConverters
+from supervisely.convert.pointcloud_episodes.kitti_360.kitti_360_helper import *
+from supervisely.convert.pointcloud_episodes.pointcloud_episodes_converter import PointcloudEpisodeConverter
+from supervisely.io.fs import (
+    file_exists,
+    get_file_name,
+    get_file_name_with_ext,
+    list_files_recursively,
+    silent_remove,
+)
+from supervisely.pointcloud_annotation.pointcloud_episode_frame_collection import PointcloudEpisodeFrameCollection
+from supervisely.pointcloud_annotation.pointcloud_episode_object_collection import PointcloudEpisodeObjectCollection
+from supervisely.pointcloud_annotation.pointcloud_episode_object import PointcloudEpisodeObject
+from supervisely.pointcloud_annotation.pointcloud_episode_frame import PointcloudEpisodeFrame
+from supervisely.pointcloud_annotation.pointcloud_figure import PointcloudFigure
+
+class KITTI360Converter(PointcloudEpisodeConverter):
+
+    class Item:
+
+        def __init__(
+            self,
+            scene_name: str,
+            frame_paths: List[str],
+            ann_data: Annotation3D,
+            poses_path: str,
+            related_images: Optional[tuple] = None,
+            custom_data: Optional[dict] = None,
+        ):
+            self._scene_name = scene_name
+            self._frame_paths = frame_paths
+            self._ann_data = ann_data
+            self._poses_path = poses_path
+            self._related_images = related_images or []
+
+            self._type = "point_cloud_episode"
+            self._custom_data = custom_data if custom_data is not None else {}
+
+    def __init__(self, *args, **kwargs):
+        self._calib_path = None
+        super().__init__(*args, **kwargs)
+
+    def __str__(self) -> str:
+        return AvailablePointcloudEpisodesConverters.KITTI360
+
+    @property
+    def key_file_ext(self) -> str:
+        return ".bin"
+
+    def validate_format(self) -> bool:
+        try:
+            import kitti360scripts
+        except ImportError:
+            logger.warn("Please run 'pip install kitti360Scripts' to import KITTI-360 data.")
+            return False
+
+        self._items = []
+        subdirs = os.listdir(self._input_data)
+        if len(subdirs) == 1:
+            self._input_data = os.path.join(self._input_data, subdirs[0])
+
+        # * Get calibration path
+        calib_dir = next(iter([(Path(path).parent).as_posix() for path in list_files_recursively(self._input_data, [".txt"], None, True) if Path(path).stem.startswith("calib")]), None)
+        if calib_dir is None:
+            return False
+        self._calib_path = calib_dir
+
+        # * Get pointcloud files paths
+        velodyne_files = list_files_recursively(self._input_data, [".bin"], None, True)
+        if len(velodyne_files) == 0:
+            return False
+
+        # * Get annotation files paths and related images
+        boxes_ann_files = list_files_recursively(self._input_data, [".xml"], None, True)
+        if len(boxes_ann_files) == 0:
+            return False
+        rimage_files = list_files_recursively(self._input_data, [".png"], None, True)
+
+        kitti_anns = []
+        for ann_file in boxes_ann_files:
+            key_name = Path(ann_file).stem
+
+            # * Get pointcloud files
+            frame_paths = []
+            for path in velodyne_files:
+                if key_name in Path(path).parts:
+                    frame_paths.append(path)
+            if len(frame_paths) == 0:
+                logger.warn("No frames found for name: %s", key_name)
+                continue
+
+            # * Get related images
+            rimages = []
+            for rimage in rimage_files:
+                path = Path(rimage)
+                if key_name in path.parts:
+                    cam_name = path.parts[-3]
+                    rimages.append((cam_name, rimage))
+
+            # * Get poses
+            poses_filter = (
+                lambda x: x.endswith("cam0_to_world.txt") and key_name in Path(x).parts
+            )
+            poses_path = next(
+                path
+                for path in list_files_recursively(self._input_data, [".txt"], None, True)
+                if poses_filter(path)
+            )
+            if poses_path is None:
+                logger.warn("No poses found for name: %s", key_name)
+                continue
+
+            # * Parse annotation
+            ann = Annotation3D(ann_file)
+            kitti_anns.append(ann)
+
+            self._items.append(
+                self.Item(key_name, frame_paths, ann, poses_path, rimages)
+            )
+
+        # * Get object class names for meta
+        obj_class_names = set()
+        for ann in kitti_anns:
+            for obj in ann.get_objects():
+                obj_class_names.add(obj.name)
+        obj_classes = [ObjClass(obj_class, Cuboid3d) for obj_class in obj_class_names]
+        self._meta = ProjectMeta(obj_classes=ObjClassCollection(obj_classes))
+        return self.items_count > 0
+
+    def to_supervisely(
+        self,
+        item,
+        meta: ProjectMeta,
+        renamed_classes: dict = {},
+        renamed_tags: dict = {},
+        static_transformations: StaticTransformations = None,
+    ) -> PointcloudEpisodeAnnotation:
+        static_transformations.set_cam2world(item._poses_path)
+
+        frame_cnt = len(item._frame_paths)
+        objs, frames = [], []
+
+        frame_idx_to_figures = {idx: [] for idx in range(frame_cnt)}
+        for obj in item._ann_data.get_objects():
+            pcd_obj = PointcloudEpisodeObject(meta.get_obj_class(obj.name))
+            objs.append(pcd_obj)
+
+            for idx in range(frame_cnt):
+                if obj.start_frame <= idx <= obj.end_frame:
+                    tr_matrix = static_transformations.world_to_velo_transformation(obj, idx)
+                    geom = convert_kitti_cuboid_to_supervisely_geometry(tr_matrix)
+                    frame_idx_to_figures[idx].append(PointcloudFigure(pcd_obj, geom, idx))
+        for idx, figures in frame_idx_to_figures.items():
+            frame = PointcloudEpisodeFrame(idx, figures)
+            frames.append(frame)
+        obj_collection = PointcloudEpisodeObjectCollection(objs)
+        frame_collection = PointcloudEpisodeFrameCollection(frames)
+        return PointcloudEpisodeAnnotation(
+            frame_cnt, objects=obj_collection, frames=frame_collection
+        )
+
+    def upload_dataset(self, api: Api, dataset_id: int, batch_size: int = 1, log_progress=True):
+        meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
+
+        dataset_info = api.dataset.get_info_by_id(dataset_id)
+        if log_progress:
+            progress, progress_cb = self.get_progress(sum([len(item._frame_paths) for item in self._items]), "Converting pointcloud episodes...")
+        else:
+            progress_cb = None
+        static_transformations = StaticTransformations(self._calib_path)
+        scene_ds = dataset_info
+        multiple_items = self.items_count > 1
+        for item in self._items:
+            scene_ds = api.dataset.create(dataset_info.project_id, item._scene_name, parent_id=dataset_id) if multiple_items else dataset_info
+            frame_to_pcd_ids = {}
+            for idx, frame_path in enumerate(item._frame_paths):
+                # * Convert pointcloud from ".bin" to ".pcd"
+                pcd_path = str(Path(frame_path).with_suffix(".pcd"))
+                if file_exists(pcd_path):
+                    logger.warning(f"Overwriting file with path: {pcd_path}")
+                convert_bin_to_pcd(frame_path, pcd_path)
+
+                # * Upload pointcloud
+                pcd_name = get_file_name_with_ext(pcd_path)
+                info = api.pointcloud_episode.upload_path(scene_ds.id, pcd_name, pcd_path, {"frame": idx})
+                pcd_id = info.id
+                frame_to_pcd_ids[idx] = pcd_id
+
+                # * Clean up
+                silent_remove(pcd_path)
+
+                if log_progress:
+                    progress_cb(1)
+
+            # * Upload photocontext
+            rimage_jsons = []
+            cam_names = []
+            hashes = api.pointcloud_episode.upload_related_images(
+                [rimage_path for _, rimage_path in item._related_images]
+            )
+            for (cam_name, rimage_path), img, pcd_id in zip(
+                item._related_images, hashes, list(frame_to_pcd_ids.values())
+            ):
+                cam_num = int(cam_name[-1])
+                rimage_info = convert_calib_to_image_meta(
+                    get_file_name(rimage_path), static_transformations, cam_num
+                )
+                image_json = {
+                    ApiField.ENTITY_ID: pcd_id,
+                    ApiField.NAME: cam_name,
+                    ApiField.HASH: img,
+                    ApiField.META: rimage_info[ApiField.META],
+                }
+                rimage_jsons.append(image_json)
+                cam_names.append(cam_name)
+            if rimage_jsons:
+                api.pointcloud_episode.add_related_images(rimage_jsons, cam_names)
+
+            # * Convert annotation and upload
+            try:
+                ann = self.to_supervisely(
+                    item, meta, renamed_classes, renamed_tags, static_transformations
+                )
+                api.pointcloud_episode.annotation.append(scene_ds.id, ann, frame_to_pcd_ids)
+            except Exception as e:
+                logger.error(
+                    f"Failed to upload annotation for scene: {scene_ds.name}. Error: {repr(e)}",
+                    stack_info=False,
+                )
+                continue
+
+            logger.info(f"Dataset ID:{scene_ds.id} has been successfully uploaded.")
+
+        if log_progress:
+            if is_development():
+                progress.close()

supervisely-6.73.322/supervisely/convert/pointcloud_episodes/kitti_360/kitti_360_helper.py

@@ -0,0 +1,386 @@
+from supervisely import logger
+from supervisely.io.fs import get_file_name
+from supervisely.geometry.cuboid_3d import Cuboid3d
+from supervisely.geometry.point_3d import Vector3d
+from supervisely.geometry.point import Point
+
+from collections import defaultdict
+import os
+import numpy as np
+
+
+MAX_N = 1000
+
+
+def local2global(semanticId, instanceId):
+    globalId = semanticId * MAX_N + instanceId
+    if isinstance(globalId, np.ndarray):
+        return globalId.astype(np.int)
+    else:
+        return int(globalId)
+
+
+def global2local(globalId):
+    semanticId = globalId // MAX_N
+    instanceId = globalId % MAX_N
+    if isinstance(globalId, np.ndarray):
+        return semanticId.astype(int), instanceId.astype(int)
+    else:
+        return int(semanticId), int(instanceId)
+
+
+annotation2global = defaultdict()
+
+
+# Abstract base class for annotation objects
+class KITTI360Object:
+    from abc import ABCMeta
+
+    __metaclass__ = ABCMeta
+
+    def __init__(self):
+        from matplotlib import cm
+
+        # the label
+        self.label = ""
+
+        # colormap
+        self.cmap = cm.get_cmap("Set1")
+        self.cmap_length = 9
+
+    def getColor(self, idx):
+        if idx == 0:
+            return np.array([0, 0, 0])
+        return np.asarray(self.cmap(idx % self.cmap_length)[:3]) * 255.0
+
+    # def assignColor(self):
+    #     from kitti360scripts.helpers.labels import id2label  # pylint: disable=import-error
+
+    #     if self.semanticId >= 0:
+    #         self.semanticColor = id2label[self.semanticId].color
+    #         if self.instanceId > 0:
+    #             self.instanceColor = self.getColor(self.instanceId)
+    #         else:
+    #             self.instanceColor = self.semanticColor
+
+
+# Class that contains the information of a single annotated object as 3D bounding box
+class KITTI360Bbox3D(KITTI360Object):
+    # Constructor
+    def __init__(self):
+        KITTI360Object.__init__(self)
+        # the polygon as list of points
+        self.vertices = []
+        self.faces = []
+        self.lines = [
+            [0, 5],
+            [1, 4],
+            [2, 7],
+            [3, 6],
+            [0, 1],
+            [1, 3],
+            [3, 2],
+            [2, 0],
+            [4, 5],
+            [5, 7],
+            [7, 6],
+            [6, 4],
+        ]
+
+        # the ID of the corresponding object
+        self.semanticId = -1
+        self.instanceId = -1
+        self.annotationId = -1
+
+        # the window that contains the bbox
+        self.start_frame = -1
+        self.end_frame = -1
+
+        # timestamp of the bbox (-1 if statis)
+        self.timestamp = -1
+
+        # projected vertices
+        self.vertices_proj = None
+        self.meshes = []
+
+        # name
+        self.name = ""
+
+    def __str__(self):
+        return self.name
+
+    # def generateMeshes(self):
+    #     self.meshes = []
+    #     if self.vertices_proj:
+    #         for fidx in range(self.faces.shape[0]):
+    #             self.meshes.append(
+    #                 [
+    #                     Point(self.vertices_proj[0][int(x)], self.vertices_proj[1][int(x)])
+    #                     for x in self.faces[fidx]
+    #                 ]
+    #             )
+
+    def parseOpencvMatrix(self, node):
+        rows = int(node.find("rows").text)
+        cols = int(node.find("cols").text)
+        data = node.find("data").text.split(" ")
+
+        mat = []
+        for d in data:
+            d = d.replace("\n", "")
+            if len(d) < 1:
+                continue
+            mat.append(float(d))
+        mat = np.reshape(mat, [rows, cols])
+        return mat
+
+    def parseVertices(self, child):
+        transform = self.parseOpencvMatrix(child.find("transform"))
+        R = transform[:3, :3]
+        T = transform[:3, 3]
+        vertices = self.parseOpencvMatrix(child.find("vertices"))
+        faces = self.parseOpencvMatrix(child.find("faces"))
+
+        vertices = np.matmul(R, vertices.transpose()).transpose() + T
+        self.vertices = vertices
+        self.faces = faces
+        self.R = R
+        self.T = T
+
+        self.transform = transform
+
+    def parseBbox(self, child):
+        from kitti360scripts.helpers.labels import kittiId2label  # pylint: disable=import-error
+
+        semanticIdKITTI = int(child.find("semanticId").text)
+        self.semanticId = kittiId2label[semanticIdKITTI].id
+        self.instanceId = int(child.find("instanceId").text)
+        # self.name = str(child.find('label').text)
+        self.name = kittiId2label[semanticIdKITTI].name
+
+        self.start_frame = int(child.find("start_frame").text)
+        self.end_frame = int(child.find("end_frame").text)
+
+        self.timestamp = int(child.find("timestamp").text)
+
+        self.annotationId = int(child.find("index").text) + 1
+
+        global annotation2global
+        annotation2global[self.annotationId] = local2global(self.semanticId, self.instanceId)
+        self.parseVertices(child)
+
+    def parseStuff(self, child):
+        from kitti360scripts.helpers.labels import name2label  # pylint: disable=import-error
+
+        classmap = {
+            "driveway": "parking",
+            "ground": "terrain",
+            "unknownGround": "ground",
+            "railtrack": "rail track",
+        }
+        label = child.find("label").text
+        if label in classmap.keys():
+            label = classmap[label]
+
+        self.start_frame = int(child.find("start_frame").text)
+        self.end_frame = int(child.find("end_frame").text)
+
+        self.semanticId = name2label[label].id
+        self.instanceId = 0
+        self.parseVertices(child)
+
+
+# Class that contains the information of the point cloud a single frame
+class KITTI360Point3D(KITTI360Object):
+    # Constructor
+    def __init__(self):
+        KITTI360Object.__init__(self)
+
+        self.vertices = []
+
+        self.vertices_proj = None
+
+        # the ID of the corresponding object
+        self.semanticId = -1
+        self.instanceId = -1
+        self.annotationId = -1
+
+        # name
+        self.name = ""
+
+        # color
+        self.semanticColor = None
+        self.instanceColor = None
+
+    def __str__(self):
+        return self.name
+
+    # def generateMeshes(self):
+    #     pass
+
+
+# Meta class for KITTI360Bbox3D
+class Annotation3D:
+    def __init__(self, labelPath):
+        from kitti360scripts.helpers.labels import labels  # pylint: disable=import-error
+        import xml.etree.ElementTree as ET
+
+        key_name = get_file_name(labelPath)
+        # load annotation
+        tree = ET.parse(labelPath)
+        root = tree.getroot()
+
+        self.objects = defaultdict(dict)
+
+        self.num_bbox = 0
+        for child in root:
+            if child.find("transform") is None:
+                continue
+            obj = KITTI360Bbox3D()
+            obj.parseBbox(child)
+            globalId = local2global(obj.semanticId, obj.instanceId)
+            self.objects[globalId][obj.timestamp] = obj
+            self.num_bbox += 1
+
+        globalIds = np.asarray(list(self.objects.keys()))
+        semanticIds, instanceIds = global2local(globalIds)
+        for label in labels:
+            if label.hasInstances:
+                print(f"{label.name:<30}:\t {(semanticIds==label.id).sum()}")
+        print(f"Loaded {len(globalIds)} instances")
+        print(f"Loaded {self.num_bbox} boxes")
+
+    def __call__(self, semanticId, instanceId, timestamp=None):
+        globalId = local2global(semanticId, instanceId)
+        if globalId in self.objects.keys():
+            # static object
+            if len(self.objects[globalId].keys()) == 1:
+                if -1 in self.objects[globalId].keys():
+                    return self.objects[globalId][-1]
+                else:
+                    return None
+            # dynamic object
+            else:
+                return self.objects[globalId][timestamp]
+        else:
+            return None
+
+    def get_objects(self):
+        return [list(obj.values())[0] for obj in self.objects.values()]
+
+class StaticTransformations:
+    def __init__(self, calibrations_path):
+        import kitti360scripts.devkits.commons.loadCalibration as lc  # pylint: disable=import-error
+
+        cam2velo_path = os.path.join(calibrations_path, "calib_cam_to_velo.txt")
+        self.cam2velo = lc.loadCalibrationRigid(cam2velo_path)
+        perspective_path = os.path.join(calibrations_path, "perspective.txt")
+        self.intrinsic_calibrations = lc.loadPerspectiveIntrinsic(perspective_path)
+        self.cam2world = None
+
+    def set_cam2world(self, cam2world_path):
+        if not os.path.isfile(cam2world_path):
+            logger.warn("Camera to world calibration file was not found")
+            return
+
+        cam2world_rows = np.loadtxt(cam2world_path)
+        cam2world_rigid = np.reshape(cam2world_rows[:, 1:], (-1, 4, 4))
+        frames_numbers = list(np.reshape(cam2world_rows[:, :1], (-1)).astype(int))
+        cam2world = {}
+
+        current_rigid = cam2world_rigid[0]
+
+        for frame_index in range(0, frames_numbers[-1]):
+            if frame_index in frames_numbers:
+                mapped_index = frames_numbers.index(frame_index)
+                current_rigid = cam2world_rigid[mapped_index]
+
+            # (Tr(cam -> world))
+            cam2world[frame_index] = current_rigid
+        self.cam2world = cam2world
+
+    def world_to_velo_transformation(self, obj, frame_index):
+        # rotate_z = Rotation.from_rotvec(np.pi * np.array([0, 0, 1])).as_matrix()
+        # rotate_z = np.hstack((rotate_z, np.asarray([[0, 0, 0]]).T))
+
+        # tr0(local -> fixed_coordinates_local)
+        tr0 = np.asarray([[0, -1, 0, 0], [1, 0, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
+
+        # tr0(fixed_coordinates_local -> world)
+        tr1 = obj.transform
+
+        # tr2(world -> cam)
+        tr2 = np.linalg.inv(self.cam2world[frame_index])
+
+        # tr3(world -> cam)
+        tr3 = self.cam2velo
+
+        return tr3 @ tr2 @ tr1 @ tr0
+
+    def get_extrinsic_matrix(self):
+        return np.linalg.inv(self.cam2velo)[:3, :4]
+
+    def get_intrinsics_matrix(self, camera_num):
+        try:
+            matrix = self.intrinsic_calibrations[f"P_rect_0{camera_num}"][:3, :3]
+            return matrix
+        except KeyError:
+            logger.warn(f"Camera {camera_num} intrinsic matrix was not found")
+        return
+
+def convert_kitti_cuboid_to_supervisely_geometry(tr_matrix):
+    import transforms3d  # pylint: disable=import-error
+    from scipy.spatial.transform.rotation import Rotation
+
+    Tdash, Rdash, Zdash, _ = transforms3d.affines.decompose44(tr_matrix)
+
+    x, y, z = Tdash[0], Tdash[1], Tdash[2]
+    position = Vector3d(x, y, z)
+
+    rotation_angles = Rotation.from_matrix(Rdash).as_euler("xyz", degrees=False)
+    r_x, r_y, r_z = rotation_angles[0], rotation_angles[1], rotation_angles[2]
+
+    # Invert the bbox by adding π to the yaw while maintaining its degree relative to the world
+    rotation = Vector3d(r_x, r_y, r_z + np.pi)
+
+    w, h, l = Zdash[0], Zdash[1], Zdash[2]
+    dimension = Vector3d(w, h, l)
+
+    return Cuboid3d(position, rotation, dimension)
+
+def convert_bin_to_pcd(src, dst):
+    import open3d as o3d  # pylint: disable=import-error
+
+    try:
+        bin = np.fromfile(src, dtype=np.float32).reshape(-1, 4)
+    except ValueError as e:
+        raise Exception(
+            f"Incorrect data in the KITTI 3D pointcloud file: {src}. "
+            f"There was an error while trying to reshape the data into a 4-column matrix: {e}. "
+            "Please ensure that the binary file contains a multiple of 4 elements to be "
+            "successfully reshaped into a (N, 4) array.\n"
+        )
+    points = bin[:, 0:3]
+    intensity = bin[:, -1]
+    intensity_fake_rgb = np.zeros((intensity.shape[0], 3))
+    intensity_fake_rgb[:, 0] = intensity
+    pc = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(points))
+    pc.colors = o3d.utility.Vector3dVector(intensity_fake_rgb)
+    o3d.io.write_point_cloud(dst, pc)
+
+
+def convert_calib_to_image_meta(image_name, static, cam_num):
+    intrinsic_matrix = static.get_intrinsics_matrix(cam_num)
+    extrinsic_matrix = static.get_extrinsic_matrix()
+
+    data = {
+        "name": image_name,
+        "meta": {
+            "deviceId": cam_num,
+            "sensorsData": {
+                "extrinsicMatrix": list(extrinsic_matrix.flatten().astype(float)),
+                "intrinsicMatrix": list(intrinsic_matrix.flatten().astype(float)),
+            },
+        },
+    }
+    return data