supervisely 6.73.263__tar.gz → 6.73.264__tar.gz

{supervisely-6.73.263/supervisely.egg-info → supervisely-6.73.264}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.263
+Version: 6.73.264
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.263 → supervisely-6.73.264}/supervisely/convert/__init__.py

@@ -37,6 +37,7 @@ from supervisely.convert.pointcloud.sly.sly_pointcloud_converter import SLYPoint
 from supervisely.convert.pointcloud.las.las_converter import LasConverter
 from supervisely.convert.pointcloud.ply.ply_converter import PlyConverter
 from supervisely.convert.pointcloud.bag.bag_converter import BagConverter
+from supervisely.convert.pointcloud.lyft.lyft_converter import LyftConverter
 
 
 # Pointcloud Episodes
@@ -44,6 +45,7 @@ from supervisely.convert.pointcloud_episodes.sly.sly_pointcloud_episodes_convert
     SLYPointcloudEpisodesConverter,
 )
 from supervisely.convert.pointcloud_episodes.bag.bag_converter import BagEpisodesConverter
+from supervisely.convert.pointcloud_episodes.lyft.lyft_converter import LyftEpisodesConverter
 
 # Video
 from supervisely.convert.video.mot.mot_converter import MOTConverter

{supervisely-6.73.263 → supervisely-6.73.264}/supervisely/convert/base_converter.py

@@ -54,11 +54,13 @@ class AvailablePointcloudConverters:
     LAS = "las/laz"
     PLY = "ply"
     BAG = "rosbag"
+    LYFT = "lyft"
 
 
 class AvailablePointcloudEpisodesConverters:
     SLY = "supervisely"
     BAG = "rosbag"
+    LYFT = "lyft"
 
 
 class AvailableVolumeConverters:

{supervisely-6.73.263 → supervisely-6.73.264}/supervisely/convert/image/image_helper.py

@@ -15,6 +15,7 @@ from supervisely.io.fs import (
     silent_remove,
 )
 
+Image.MAX_IMAGE_PIXELS = None
 EXT_TO_CONVERT = [".heic", ".avif", ".heif", ".jfif"]
 
 

supervisely-6.73.264/supervisely/convert/pointcloud/lyft/lyft_converter.py

@@ -0,0 +1,287 @@
+import numpy as np
+from pathlib import Path
+from typing import Dict, Optional
+
+from supervisely import (
+    Api,
+    ObjClass,
+    PointcloudAnnotation,
+    ProjectMeta,
+    logger,
+    is_development,
+    Progress,
+    PointcloudObject,
+    TagMeta,
+    TagValueType,
+)
+from supervisely.io import fs
+from supervisely.convert.base_converter import AvailablePointcloudConverters
+from supervisely.convert.pointcloud.pointcloud_converter import PointcloudConverter
+from supervisely.geometry.cuboid_3d import Cuboid3d
+from supervisely.convert.pointcloud.lyft import lyft_helper
+from supervisely.api.api import ApiField
+from datetime import datetime
+from supervisely import TinyTimer
+from supervisely.pointcloud_annotation.pointcloud_annotation import (
+    PointcloudFigure,
+    PointcloudObjectCollection,
+    PointcloudTagCollection,
+)
+from supervisely.pointcloud_annotation.pointcloud_tag import PointcloudTag
+
+
+class LyftConverter(PointcloudConverter):
+    class Item(PointcloudConverter.Item):
+
+        def __init__(
+            self,
+            item_path,
+            ann_data: str = None,
+            related_images: list = None,
+            custom_data: dict = None,
+            scene_name: str = None,
+        ):
+            super().__init__(item_path, ann_data, related_images, custom_data)
+            self._type = "point_cloud"
+            self._scene_name = scene_name
+
+    def __init__(
+        self,
+        input_data: str,
+        labeling_interface: str,
+        upload_as_links: bool,
+        remote_files_map: Optional[Dict[str, str]] = None,
+    ):
+        super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
+        self._is_pcd_episode = False
+        self._lyft = None
+
+    def __str__(self) -> str:
+        return AvailablePointcloudConverters.LYFT
+
+    @property
+    def key_file_ext(self) -> str:
+        return ".bin"
+
+    def validate_format(self) -> bool:
+        try:
+            from lyft_dataset_sdk.lyftdataset import LyftDataset as Lyft
+        except ImportError:
+            logger.warn(
+                'Install "lyft_dataset_sdk" python package to import datasets in LYFT format.'
+            )
+            return False
+
+        def filter_fn(path):
+            return all([(Path(path) / name).exists() for name in lyft_helper.FOLDER_NAMES])
+
+        input_paths = [d for d in fs.dirs_filter(self._input_data, filter_fn)]
+        if len(input_paths) == 0:
+            return False
+        input_path = input_paths[0]
+
+        lidar_dir = input_path + "/lidar/"
+        json_dir = input_path + "/data/"
+        if lyft_helper.validate_ann_dir(json_dir) is False:
+            return False
+
+        bin_files = fs.list_files_recursively(
+            lidar_dir, [self.key_file_ext], ignore_valid_extensions_case=True
+        )
+
+        if len(bin_files) == 0:
+            return False
+
+        # check if pointclouds have 5 columns (x, y, z, intensity, ring)
+        pointcloud = np.fromfile(bin_files[0], dtype=np.float32)
+        if pointcloud.shape[0] % 5 != 0:
+            return False
+
+        try:
+            t = TinyTimer()
+            lyft = Lyft(data_path=input_path, json_path=json_dir, verbose=False)
+            self._lyft: Lyft = lyft
+            logger.info(f"LyftDataset initialization took {t.get_sec():.2f} sec")
+        except Exception as e:
+            logger.info(f"Failed to initialize LyftDataset: {e}")
+            return False
+
+        t = TinyTimer()
+        progress = Progress(f"Extracting annotations from available scenes...")
+        # i = 0 # for debug
+        for scene in lyft_helper.get_available_scenes(lyft):
+            scene_name = scene["name"]
+            sample_datas = lyft_helper.extract_data_from_scene(lyft, scene)
+            if sample_datas is None:
+                logger.warning(f"Failed to extract sample data from scene: {scene['name']}.")
+                continue
+            for sample_data in sample_datas:
+                item_path = sample_data["lidar_path"]
+                ann_data = sample_data["ann_data"]
+                related_images = lyft_helper.get_related_images(ann_data)
+                custom_data = sample_data.get("custom_data", {})
+                item = self.Item(item_path, ann_data, related_images, custom_data, scene_name)
+                self._items.append(item)
+            progress.iter_done_report()
+            # i += 1
+            # if i == 2:
+            #     break
+        t = t.get_sec()
+        logger.info(
+            f"Lyft annotation extraction took {t:.2f} sec ({(t / self.items_count):.3f} sec per sample)"
+        )
+
+        return self.items_count > 0
+
+    def to_supervisely(
+        self,
+        item: PointcloudConverter.Item,
+        meta: ProjectMeta,
+        renamed_classes: dict = {},
+        renamed_tags: dict = {},
+    ) -> PointcloudAnnotation:
+        """
+        Converts a point cloud item and its annotations to the supervisely formats.
+
+        Args:
+            item (PointcloudConverter.Item): The point cloud item to convert.
+            meta (ProjectMeta): The project meta.
+
+        Returns:
+            PointcloudAnnotation: The converted point cloud annotation.
+        """
+        import open3d as o3d  # pylint: disable=import-error
+
+        if getattr(item, "ann_data", None) is None:
+            return PointcloudAnnotation()
+
+        data = item.ann_data
+
+        # * Convert annotation to json
+        boxes = data["gt_boxes"]
+        names = data["names"]
+
+        objects = []
+        for name, box in zip(names, boxes):
+            center = [float(box[0]), float(box[1]), float(box[2])]
+            size = [float(box[3]), float(box[5]), float(box[4])]
+            ry = float(box[6])
+
+            yaw = ry - np.pi
+            yaw = yaw - np.floor(yaw / (2 * np.pi) + 0.5) * 2 * np.pi
+            world_cam = None
+            objects.append(o3d.ml.datasets.utils.BEVBox3D(center, size, yaw, name, -1.0, world_cam))
+            objects[-1].yaw = ry
+
+        geoms = [lyft_helper._convert_BEVBox3D_to_geometry(box) for box in objects]
+
+        figures = []
+        objs = []
+        for l, geometry, token in zip(
+            objects, geoms, data["instance_tokens"]
+        ):  # by object in point cloud
+            class_name = renamed_classes.get(l.label_class, l.label_class)
+            tag_names = [
+                self._lyft.get("attribute", attr_token).get("name", None)
+                for attr_token in token["attribute_tokens"]
+            ]
+            tag_col = None
+            if len(tag_names) > 0 and all([tag_name is not None for tag_name in tag_names]):
+                tag_meta_names = [renamed_tags.get(name, name) for name in tag_names]
+                tag_metas = [meta.get_tag_meta(tag_meta_name) for tag_meta_name in tag_meta_names]
+                tag_col = PointcloudTagCollection([PointcloudTag(meta, None) for meta in tag_metas])
+            pcobj = PointcloudObject(meta.get_obj_class(class_name), tag_col)
+            figures.append(PointcloudFigure(pcobj, geometry))
+            objs.append(pcobj)
+        return PointcloudAnnotation(PointcloudObjectCollection(objs), figures)
+
+    def upload_dataset(self, api: Api, dataset_id: int, batch_size: int = 1, log_progress=True):
+        unique_names = {name for item in self._items for name in item.ann_data["names"]}
+        tag_names = {tag["name"] for tag in self._lyft.attribute}
+        self._meta = ProjectMeta(
+            [ObjClass(name, Cuboid3d) for name in unique_names],
+            [TagMeta(tag, TagValueType.NONE) for tag in tag_names],
+        )
+        meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
+
+        scene_names = set([item._scene_name for item in self._items])
+        dataset_info = api.dataset.get_info_by_id(dataset_id)
+        scene_name_to_dataset = {}
+
+        multiple_scenes = len(scene_names) > 1
+        if multiple_scenes:
+            logger.info(
+                f"Found {len(scene_names)} scenes ({self.items_count} pointclouds) in the input data."
+            )
+            # * Create a nested dataset for each scene
+            for name in scene_names:
+                ds = api.dataset.create(
+                    dataset_info.project_id,
+                    name,
+                    change_name_if_conflict=True,
+                    parent_id=dataset_id,
+                )
+                scene_name_to_dataset[name] = ds
+        else:
+            scene_name_to_dataset[list(scene_names)[0]] = dataset_info
+
+        if log_progress:
+            progress, progress_cb = self.get_progress(self.items_count, "Converting pointclouds...")
+        else:
+            progress_cb = None
+
+        for item in self._items:
+            # * Get the current dataset for the scene
+            current_dataset = scene_name_to_dataset.get(item._scene_name, None)
+            if current_dataset is None:
+                raise RuntimeError(f"Dataset not found for scene name: {item._scene_name}")
+            current_dataset_id = current_dataset.id
+
+            # * Convert timestamp to ISO format
+            iso_time = datetime.utcfromtimestamp(item.ann_data["timestamp"] / 1e6).isoformat() + "Z"
+            item.ann_data["timestamp"] = iso_time
+
+            # * Convert pointcloud from ".bin" to ".pcd"
+            pcd_path = str(Path(item.path).with_suffix(".pcd"))
+            if fs.file_exists(pcd_path):
+                logger.warning(f"Overwriting file with path: {pcd_path}")
+            lyft_helper.convert_bin_to_pcd(item.path, pcd_path)
+
+            # * Upload pointcloud
+            pcd_name = fs.get_file_name(pcd_path)
+            info = api.pointcloud.upload_path(current_dataset_id, pcd_name, pcd_path, {})
+            pcd_id = info.id
+
+            # * Convert annotation and upload
+            ann = self.to_supervisely(item, meta, renamed_classes, renamed_tags)
+            api.pointcloud.annotation.append(pcd_id, ann)
+
+            # * Upload related images
+            image_jsons = []
+            camera_names = []
+            for img_path, rimage_info in lyft_helper.generate_rimage_infos(
+                item._related_images, item.ann_data
+            ):
+                img = api.pointcloud.upload_related_image(img_path)
+                image_jsons.append(
+                    {
+                        ApiField.ENTITY_ID: pcd_id,
+                        ApiField.NAME: rimage_info[ApiField.NAME],
+                        ApiField.HASH: img,
+                        ApiField.META: rimage_info[ApiField.META],
+                    }
+                )
+                camera_names.append(rimage_info[ApiField.META]["deviceId"])
+            if len(image_jsons) > 0:
+                api.pointcloud.add_related_images(image_jsons, camera_names)
+
+            # * Clean up
+            fs.silent_remove(pcd_path)
+            if log_progress:
+                progress_cb(1)
+
+        logger.info(f"Dataset ID:{current_dataset_id} has been successfully uploaded.")
+
+        if log_progress:
+            if is_development():
+                progress.close()

supervisely-6.73.264/supervisely/convert/pointcloud/lyft/lyft_helper.py

@@ -0,0 +1,231 @@
+import os
+from typing import List, Tuple
+import numpy as np
+from supervisely import logger, fs
+from supervisely.geometry.cuboid_3d import Cuboid3d, Vector3d
+
+TABLE_NAMES = [
+    "category",
+    "attribute",
+    "visibility",
+    "instance",
+    "sensor",
+    "calibrated_sensor",
+    "ego_pose",
+    "log",
+    "scene",
+    "sample",
+    "sample_data",
+    "sample_annotation",
+    "map",
+]
+FOLDER_NAMES = ["data", "lidar", "images", "maps"]
+
+
+def get_available_scenes(lyft):
+    for scene in lyft.scene:
+        token = scene["token"]
+        scene_rec = lyft.get("scene", token)
+        sample_rec = lyft.get("sample", scene_rec["first_sample_token"])
+        sample_data = lyft.get("sample_data", sample_rec["data"]["LIDAR_TOP"])
+
+        lidar_path, boxes, _ = lyft.get_sample_data(sample_data["token"])
+        if not os.path.exists(str(lidar_path)):
+            continue
+        yield scene
+
+
+def extract_data_from_scene(lyft, scene):
+    try:
+        from pyquaternion import Quaternion
+        from lyft_dataset_sdk.utils.geometry_utils import transform_matrix
+    except ImportError as ie:
+        logger.warn(f"Lazy import failed. Error: {ie}")
+        return
+
+    new_token = scene["first_sample_token"]
+    dataset_data = []
+    num_samples = scene["nbr_samples"]
+    for i in range(num_samples):
+        my_sample = lyft.get("sample", new_token)
+
+        data = {}
+        data["ann_data"] = {}
+        data["ann_data"]["timestamp"] = my_sample["timestamp"]
+        data["custom_data"] = lyft.get("log", scene["log_token"])
+
+        sensor_token = my_sample["data"]["LIDAR_TOP"]
+        lidar_path, boxes, _ = lyft.get_sample_data(sensor_token)
+        if not os.path.exists(str(lidar_path)):
+            logger.debug(f"Skipping sample {new_token} - lidar file doesn't exist")
+            continue
+        data["lidar_path"] = str(lidar_path)
+
+        sd_record_lid = lyft.get("sample_data", sensor_token)
+        cs_record_lid = lyft.get("calibrated_sensor", sd_record_lid["calibrated_sensor_token"])
+        ego_record_lid = lyft.get("ego_pose", sd_record_lid["ego_pose_token"])
+
+        locs = np.array([b.center for b in boxes]).reshape(-1, 3)
+        dims = np.array([b.wlh for b in boxes]).reshape(-1, 3)
+        rots = np.array([b.orientation.yaw_pitch_roll[0] for b in boxes]).reshape(-1, 1)
+
+        gt_boxes = np.concatenate([locs, dims, -rots + np.pi / 2], axis=1)
+        names = np.array([b.name for b in boxes])
+        data["ann_data"]["names"] = names
+        data["ann_data"]["gt_boxes"] = gt_boxes
+        instance_tokens = [lyft.get("sample_annotation", box.token) for box in boxes]
+        data["ann_data"]["instance_tokens"] = instance_tokens
+
+        for sensor, sensor_token in my_sample["data"].items():
+            if "CAM" in sensor:
+                img_path, boxes, cam_intrinsic = lyft.get_sample_data(sensor_token)
+                if not os.path.exists(img_path):
+                    logger.debug(f"pass {sensor} - image doesn't exist")
+                    continue
+                data["ann_data"][sensor] = str(img_path)
+
+                sd_record_cam = lyft.get("sample_data", sensor_token)
+                cs_record_cam = lyft.get(
+                    "calibrated_sensor", sd_record_cam["calibrated_sensor_token"]
+                )
+                ego_record_cam = lyft.get("ego_pose", sd_record_cam["ego_pose_token"])
+                cam_height = sd_record_cam["height"]
+                cam_width = sd_record_cam["width"]
+
+                lid_to_ego = transform_matrix(
+                    cs_record_lid["translation"],
+                    Quaternion(cs_record_lid["rotation"]),
+                    inverse=False,
+                )
+                lid_ego_to_world = transform_matrix(
+                    ego_record_lid["translation"],
+                    Quaternion(ego_record_lid["rotation"]),
+                    inverse=False,
+                )
+                world_to_cam_ego = transform_matrix(
+                    ego_record_cam["translation"],
+                    Quaternion(ego_record_cam["rotation"]),
+                    inverse=True,
+                )
+                ego_to_cam = transform_matrix(
+                    cs_record_cam["translation"],
+                    Quaternion(cs_record_cam["rotation"]),
+                    inverse=True,
+                )
+
+                velo_to_cam = np.dot(
+                    ego_to_cam, np.dot(world_to_cam_ego, np.dot(lid_ego_to_world, lid_to_ego))
+                )
+                velo_to_cam_rot = velo_to_cam[:3, :3]
+                velo_to_cam_trans = velo_to_cam[:3, 3]
+
+                data["ann_data"][f"{sensor}_extrinsic"] = np.hstack(
+                    (velo_to_cam_rot, velo_to_cam_trans.reshape(3, 1))
+                )
+                data["ann_data"][f"{sensor}_intrinsic"] = np.asarray(
+                    cs_record_cam["camera_intrinsic"]
+                )
+                data["ann_data"][f"{sensor}_imsize"] = (cam_width, cam_height)
+            else:
+                logger.debug(f"pass {sensor} - isn't a camera")
+        dataset_data.append(data)
+
+        new_token = my_sample.get("next")
+        if not new_token:
+            break
+
+    return dataset_data
+
+
+def generate_rimage_infos(related_images: List[Tuple[str, str]], ann_data):
+    sensors_to_skip = ["_intrinsic", "_extrinsic", "_imsize"]
+    for sensor, image_path in related_images:
+        if not any([sensor.endswith(s) for s in sensors_to_skip]):
+            image_name = fs.get_file_name_with_ext(image_path)
+            sly_path_img = os.path.join(os.path.dirname(image_path), image_name)
+            img_info = {
+                "name": image_name,
+                "meta": {
+                    "deviceId": sensor,
+                    "timestamp": ann_data["timestamp"],
+                    "sensorsData": {
+                        "extrinsicMatrix": list(
+                            ann_data[f"{sensor}_extrinsic"].flatten().astype(float)
+                        ),
+                        "intrinsicMatrix": list(
+                            ann_data[f"{sensor}_intrinsic"].flatten().astype(float)
+                        ),
+                    },
+                },
+            }
+            yield sly_path_img, img_info
+
+
+def _convert_BEVBox3D_to_geometry(box):
+    bbox = box.to_xyzwhlr()
+    dim = bbox[[3, 5, 4]]
+    pos = bbox[:3] + [0, 0, dim[1] / 2]
+    yaw = bbox[-1]
+    position = Vector3d(float(pos[0]), float(pos[1]), float(pos[2]))
+    rotation = Vector3d(0, 0, float(-yaw))
+
+    dimension = Vector3d(float(dim[0]), float(dim[2]), float(dim[1]))
+    geometry = Cuboid3d(position, rotation, dimension)
+    return geometry
+
+
+def convert_bin_to_pcd(bin_file, save_filepath):
+    import open3d as o3d  # pylint: disable=import-error
+
+    b = np.fromfile(bin_file, dtype=np.float32).reshape(-1, 5)
+    points = b[:, 0:3]
+    intensity = b[:, 3]
+    ring_index = b[:, 4]
+    intensity_fake_rgb = np.zeros((intensity.shape[0], 3))
+    intensity_fake_rgb[:, 0] = (
+        intensity  # red The intensity measures the reflectivity of the objects
+    )
+    intensity_fake_rgb[:, 1] = (
+        ring_index  # green ring index is the index of the laser ranging from 0 to 31
+    )
+
+    pc = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(points))
+    pc.colors = o3d.utility.Vector3dVector(intensity_fake_rgb)
+    o3d.io.write_point_cloud(save_filepath, pc)
+
+
+def get_related_images(ann_data):
+    # sensors_to_skip = ["_intrinsic", "_extrinsic", "_imsize"]
+    # return [
+    #     (sensor, img_path)
+    #     for sensor, img_path in ann_data.items()
+    #     if "CAM" in sensor and not any([sensor.endswith(s) for s in sensors_to_skip])
+    # ]
+    return [(sensor, img_path) for sensor, img_path in ann_data.items() if "CAM" in sensor]
+
+
+def validate_ann_dir(ann_dir):
+    if any([not fs.file_exists(f"{ann_dir}/{d}.json") for d in TABLE_NAMES]):
+        return False
+    return True
+
+
+def lyft_annotation_to_BEVBox3D(data):
+    import open3d as o3d  # pylint: disable=import-error
+
+    boxes = data["gt_boxes"]
+    names = data["names"]
+
+    objects = []
+    for name, box in zip(names, boxes):
+        center = [float(box[0]), float(box[1]), float(box[2])]
+        size = [float(box[3]), float(box[5]), float(box[4])]
+        ry = float(box[6])
+
+        yaw = ry - np.pi
+        yaw = yaw - np.floor(yaw / (2 * np.pi) + 0.5) * 2 * np.pi
+        world_cam = None
+        objects.append(o3d.ml.datasets.utils.BEVBox3D(center, size, yaw, name, -1.0, world_cam))
+        objects[-1].yaw = ry
+
+    return objects