supervisely 6.73.461__py3-none-any.whl → 6.73.470__py3-none-any.whl

supervisely/app/widgets/table/table.py

@@ -1,3 +1,5 @@
+# isort: skip_file
+
 import copy
 import io
 
@@ -54,9 +56,8 @@ class PackerUnpacker:
 
     @staticmethod
     def pandas_unpacker(data: pd.DataFrame):
-        data = data.replace({np.nan: None})
-        # data = data.astype(object).replace(np.nan, "-") # TODO: replace None later
-
+        # Keep None/NaN values in source data, don't replace them
+        # They will be converted to "" only when sending to frontend
         unpacked_data = {
             "columns": data.columns.to_list(),
             "data": data.values.tolist(),
@@ -169,9 +170,35 @@ class Table(Widget):
 
         super().__init__(widget_id=widget_id, file_path=__file__)
 
+    def _prepare_data_for_frontend(self, data_dict):
+        """Convert None and NaN values to empty strings for frontend display.
+        This preserves the original None/NaN values in _parsed_data.
+        """
+        import math
+
+        display_data = copy.deepcopy(data_dict)
+
+        # Convert None/NaN in data rows
+        for row in display_data.get("data", []):
+            for i in range(len(row)):
+                value = row[i]
+                # Check for None or NaN (NaN is a float that doesn't equal itself)
+                if value is None or (isinstance(value, float) and math.isnan(value)):
+                    row[i] = ""
+
+        # Convert None/NaN in summary row if present
+        if "summaryRow" in display_data and display_data["summaryRow"] is not None:
+            summary_row = display_data["summaryRow"]
+            for i in range(len(summary_row)):
+                value = summary_row[i]
+                if value is None or (isinstance(value, float) and math.isnan(value)):
+                    summary_row[i] = ""
+
+        return display_data
+
     def get_json_data(self):
         return {
-            "table_data": self._parsed_data,
+            "table_data": self._prepare_data_for_frontend(self._parsed_data),
             "table_options": {
                 "perPage": self._per_page,
                 "pageSizes": self._page_sizes,
@@ -255,13 +282,17 @@ class Table(Widget):
 
     def read_json(self, value: dict) -> None:
         self._update_table_data(input_data=value)
-        DataJson()[self.widget_id]["table_data"] = self._parsed_data
+        DataJson()[self.widget_id]["table_data"] = self._prepare_data_for_frontend(
+            self._parsed_data
+        )
         DataJson().send_changes()
         self.clear_selection()
 
     def read_pandas(self, value: pd.DataFrame) -> None:
         self._update_table_data(input_data=value)
-        DataJson()[self.widget_id]["table_data"] = self._parsed_data
+        DataJson()[self.widget_id]["table_data"] = self._prepare_data_for_frontend(
+            self._parsed_data
+        )
         DataJson().send_changes()
         self.clear_selection()
 
@@ -272,7 +303,9 @@ class Table(Widget):
         index = len(table_data) if index > len(table_data) or index < 0 else index
 
         self._parsed_data["data"].insert(index, data)
-        DataJson()[self.widget_id]["table_data"] = self._parsed_data
+        DataJson()[self.widget_id]["table_data"] = self._prepare_data_for_frontend(
+            self._parsed_data
+        )
         DataJson().send_changes()
 
     def pop_row(self, index=-1):
@@ -284,7 +317,9 @@ class Table(Widget):
 
         if len(self._parsed_data["data"]) != 0:
             popped_row = self._parsed_data["data"].pop(index)
-            DataJson()[self.widget_id]["table_data"] = self._parsed_data
+            DataJson()[self.widget_id]["table_data"] = self._prepare_data_for_frontend(
+                self._parsed_data
+            )
             DataJson().send_changes()
             return popped_row
 
@@ -382,11 +417,27 @@ class Table(Widget):
         StateJson()[self.widget_id]["selected_row"] = {}
         StateJson().send_changes()
 
+    @staticmethod
+    def _values_equal(val1, val2):
+        """Compare two values, handling NaN specially."""
+        import math
+
+        # Check if both are NaN
+        is_nan1 = isinstance(val1, float) and math.isnan(val1)
+        is_nan2 = isinstance(val2, float) and math.isnan(val2)
+        if is_nan1 and is_nan2:
+            return True
+        # Check if both are None
+        if val1 is None and val2 is None:
+            return True
+        # Regular comparison
+        return val1 == val2
+
     def delete_row(self, key_column_name, key_cell_value):
         col_index = self._parsed_data["columns"].index(key_column_name)
         row_indices = []
         for idx, row in enumerate(self._parsed_data["data"]):
-            if row[col_index] == key_cell_value:
+            if self._values_equal(row[col_index], key_cell_value):
                 row_indices.append(idx)
         if len(row_indices) == 0:
             raise ValueError('Column "{key_column_name}" does not have value "{key_cell_value}"')
@@ -400,7 +451,7 @@ class Table(Widget):
         key_col_index = self._parsed_data["columns"].index(key_column_name)
         row_indices = []
         for idx, row in enumerate(self._parsed_data["data"]):
-            if row[key_col_index] == key_cell_value:
+            if self._values_equal(row[key_col_index], key_cell_value):
                 row_indices.append(idx)
         if len(row_indices) == 0:
             raise ValueError('Column "{key_column_name}" does not have value "{key_cell_value}"')
@@ -411,20 +462,24 @@ class Table(Widget):
 
         col_index = self._parsed_data["columns"].index(column_name)
         self._parsed_data["data"][row_indices[0]][col_index] = new_value
-        DataJson()[self.widget_id]["table_data"] = self._parsed_data
+        DataJson()[self.widget_id]["table_data"] = self._prepare_data_for_frontend(
+            self._parsed_data
+        )
         DataJson().send_changes()
 
     def update_matching_cells(self, key_column_name, key_cell_value, column_name, new_value):
         key_col_index = self._parsed_data["columns"].index(key_column_name)
         row_indices = []
         for idx, row in enumerate(self._parsed_data["data"]):
-            if row[key_col_index] == key_cell_value:
+            if self._values_equal(row[key_col_index], key_cell_value):
                 row_indices.append(idx)
 
         col_index = self._parsed_data["columns"].index(column_name)
         for row_idx in row_indices:
             self._parsed_data["data"][row_idx][col_index] = new_value
-        DataJson()[self.widget_id]["table_data"] = self._parsed_data
+        DataJson()[self.widget_id]["table_data"] = self._prepare_data_for_frontend(
+            self._parsed_data
+        )
         DataJson().send_changes()
 
     def sort(self, column_id: int = None, direction: Optional[Literal["asc", "desc"]] = None):

supervisely/convert/pointcloud/nuscenes_conv/nuscenes_converter.py

@@ -1,9 +1,10 @@
 import os
+import uuid
 from typing import Dict, List, Optional
 
 import supervisely.convert.pointcloud_episodes.nuscenes_conv.nuscenes_helper as helpers
 import supervisely.io.fs as fs
-from supervisely import PointcloudAnnotation, PointcloudObject
+from supervisely import KeyIdMap, PointcloudAnnotation, PointcloudObject
 from supervisely._utils import is_development
 from supervisely.annotation.obj_class import ObjClass
 from supervisely.annotation.tag_meta import TagMeta, TagValueType
@@ -30,19 +31,6 @@ from supervisely.sly_logger import logger
 class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
     """Converter for NuScenes pointcloud format."""
 
-    def __init__(
-        self,
-        input_data: str,
-        labeling_interface: str,
-        upload_as_links: bool,
-        remote_files_map: Optional[Dict[str, str]] = None,
-    ):
-        super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
-        self._nuscenes = None
-
-    def __str__(self) -> str:
-        return AvailablePointcloudConverters.NUSCENES
-
     def to_supervisely(
         self,
         scene_sample: helpers.Sample,
@@ -69,14 +57,21 @@ class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
         return PointcloudAnnotation(PointcloudObjectCollection(objs), figures)
 
     def upload_dataset(self, api: Api, dataset_id: int, batch_size: int = 1, log_progress=True):
-        nuscenes = self._nuscenes
+        from nuscenes.nuscenes import NuScenes  # pylint: disable=import-error
+
+        nuscenes: NuScenes = self._nuscenes
+
+        key_id_map = KeyIdMap()
 
         tag_metas = [TagMeta(attr["name"], TagValueType.NONE) for attr in nuscenes.attribute]
         obj_classes = []
+        classes_token_map = {}
         for category in nuscenes.category:
             color = nuscenes.colormap[category["name"]]
             description = helpers.trim_description(category["description"])
             obj_classes.append(ObjClass(category["name"], Cuboid3d, color, description=description))
+            classes_token_map[category["token"]] = category["name"]
+        self._custom_data["classes_token_map"] = classes_token_map
 
         self._meta = ProjectMeta(obj_classes, tag_metas)
         meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
@@ -108,6 +103,7 @@ class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
         else:
             progress_cb = None
 
+        self._custom_data["frame_token_map"] = {}
         for scene in nuscenes.scene:
             current_dataset_id = scene_name_to_dataset[scene["name"]].id
 
@@ -116,8 +112,10 @@ class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
 
             # * Extract scene's samples
             scene_samples: List[helpers.Sample] = []
+            frame_token_map = {}
             for i in range(scene["nbr_samples"]):
                 sample = nuscenes.get("sample", sample_token)
+                frame_token_map[sample["token"]] = i
                 lidar_path, boxes, _ = nuscenes.get_sample_data(sample["data"]["LIDAR_TOP"])
                 if not os.path.exists(lidar_path):
                     logger.warning(f'Scene "{scene["name"]}" has no LIDAR data.')
@@ -137,9 +135,11 @@ class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
                     ]
                     visibility = nuscenes.get("visibility", ann["visibility_token"])["level"]
 
+                    ann_uuid = uuid.UUID(ann["token"])
                     ann = helpers.AnnotationObject(
                         name=name,
                         bbox=box,
+                        token=ann_uuid,
                         instance_token=current_instance_token,
                         parent_token=parent_token,
                         category=category,
@@ -162,6 +162,7 @@ class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
                 ]
                 scene_samples.append(helpers.Sample(timestamp, lidar_path, anns, camera_data))
                 sample_token = sample["next"]
+            self._custom_data["frame_token_map"][current_dataset_id] = frame_token_map
 
             # * Convert and upload pointclouds w/ annotations
             for idx, sample in enumerate(scene_samples):
@@ -182,7 +183,7 @@ class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
                 pcd_id = info.id
                 # * Upload pointcloud annotation
                 try:
-                    api.pointcloud.annotation.append(pcd_id, pcd_ann)
+                    api.pointcloud.annotation.append(pcd_id, pcd_ann, key_id_map)
                 except Exception as e:
                     error_msg = getattr(getattr(e, "response", e), "text", str(e))
                     logger.warning(
@@ -213,6 +214,16 @@ class NuscenesConverter(NuscenesEpisodesConverter, PointcloudConverter):
 
             logger.info(f"Dataset ID:{current_dataset_id} has been successfully uploaded.")
 
+        key_id_map = key_id_map.to_dict()
+        key_id_map.pop("tags")
+        key_id_map.pop("videos")
+        self._custom_data["key_id_map"] = key_id_map
+
+        project_id = dataset_info.project_id
+        current_custom_data = api.project.get_custom_data(project_id)
+        current_custom_data.update(self._custom_data)
+        api.project.update_custom_data(project_id, current_custom_data)
+
         if log_progress:
             if is_development():
                 progress.close()

supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_converter.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 import os
+import uuid
 from os import path as osp
 from pathlib import Path
 from typing import Dict, List, Optional
@@ -37,19 +40,14 @@ from supervisely.pointcloud_annotation.pointcloud_figure import PointcloudFigure
 from supervisely.project.project_meta import ProjectMeta
 from supervisely.sly_logger import logger
 from supervisely.tiny_timer import TinyTimer
+from supervisely.video_annotation.key_id_map import KeyIdMap
 
 
 class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
     """Converter for NuScenes pointcloud episodes format."""
 
-    def __init__(
-        self,
-        input_data: str,
-        labeling_interface: str,
-        upload_as_links: bool,
-        remote_files_map: Optional[Dict[str, str]] = None,
-    ):
-        super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
+    _nuscenes: "NuScenes" = None  # type: ignore
+    _custom_data: Dict = {}
 
     def __str__(self) -> str:
         return AvailablePointcloudConverters.NUSCENES
@@ -61,8 +59,10 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
             logger.warning("Please, run 'pip install nuscenes-devkit' to import NuScenes data.")
             return False
 
-        def _contains_tables(p):
-            return all(fs.file_exists(Path(p) / f"{name}.json") for name in helpers.TABLE_NAMES)
+        table_json_filenames = [f"{name}.json" for name in helpers.TABLE_NAMES]
+
+        def _contains_tables(dir_path: str) -> bool:
+            return all(fs.file_exists(osp.join(dir_path, table)) for table in table_json_filenames)
 
         def _filter_fn(path):
             has_tables = False
@@ -89,11 +89,13 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
             logger.debug(f"Failed to initialize NuScenes: {e}")
             return False
 
+        self._custom_data["nuscenes_version"] = version
+        self._custom_data["dataroot"] = input_path
         return True
 
     def to_supervisely(
         self,
-        scene_samples: List[helpers.Sample],
+        scene_samples: Dict[str, helpers.Sample],
         meta: ProjectMeta,
         renamed_classes: dict = {},
         renamed_tags: dict = {},
@@ -101,9 +103,13 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
         token_to_obj = {}
         frames = []
         tags = []
-        for sample_i, sample in enumerate(scene_samples):
+        frame_idx_to_scene_sample_token = {}
+        if "frame_token_map" not in self._custom_data:
+            self._custom_data["frame_token_map"] = {}
+        for sample_i, (token, sample) in enumerate(scene_samples.items()):
             figures = []
             for obj in sample.anns:
+                ann_token = uuid.UUID(obj.token)
                 instance_token = obj.instance_token
                 class_name = obj.category
                 parent_obj_token = obj.parent_token
@@ -113,7 +119,9 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
                     obj_class_name = renamed_classes.get(class_name, class_name)
                     obj_class = meta.get_obj_class(obj_class_name)
                     obj_tags = None  # ! TODO: fix tags
-                    pcd_ep_obj = PointcloudEpisodeObject(obj_class, obj_tags)
+                    pcd_ep_obj = PointcloudEpisodeObject(
+                        obj_class, obj_tags, uuid.UUID(instance_token)
+                    )
                     # * Assign the object to the starting token
                     token_to_obj[instance_token] = pcd_ep_obj
                     parent_object = pcd_ep_obj
@@ -122,29 +130,41 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
                     token_to_obj[instance_token] = token_to_obj[parent_obj_token]
                     parent_object = token_to_obj[parent_obj_token]
                 geom = obj.to_supervisely()
-                pcd_figure = PointcloudFigure(parent_object, geom, sample_i)
+                pcd_figure = PointcloudFigure(parent_object, geom, sample_i, ann_token)
                 figures.append(pcd_figure)
+                frame_idx_to_scene_sample_token[sample_i] = token
             frame = PointcloudEpisodeFrame(sample_i, figures)
             frames.append(frame)
         tag_collection = PointcloudEpisodeTagCollection(tags) if len(tags) > 0 else None
+        self._custom_data["frame_token_map"][self._current_ds_id] = frame_idx_to_scene_sample_token
+        key_uuid = uuid.UUID(token)
         return PointcloudEpisodeAnnotation(
             len(frames),
             PointcloudEpisodeObjectCollection(list(set(token_to_obj.values()))),
             PointcloudEpisodeFrameCollection(frames),
             tag_collection,
+            key=key_uuid,
         )
 
     def upload_dataset(self, api: Api, dataset_id: int, batch_size: int = 1, log_progress=True):
-        nuscenes = self._nuscenes
+        from nuscenes import NuScenes  # pylint: disable=import-error
+
+        nuscenes: NuScenes = self._nuscenes
+        key_id_map = KeyIdMap()
 
         tag_metas = [TagMeta(attr["name"], TagValueType.NONE) for attr in nuscenes.attribute]
-        obj_classes = []
+        obj_classes = {}
         for category in nuscenes.category:
             color = nuscenes.colormap[category["name"]]
             description = helpers.trim_description(category.get("description", ""))
-            obj_classes.append(ObjClass(category["name"], Cuboid3d, color, description=description))
+            token = category["token"]
+            obj_classes[token] = ObjClass(
+                category["name"], Cuboid3d, color, description=description
+            )
+
+        self._custom_data["classes_token_map"] = {k: v.name for k, v in obj_classes.items()}
 
-        self._meta = ProjectMeta(obj_classes, tag_metas)
+        self._meta = ProjectMeta(list(obj_classes.values()), tag_metas)
         meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
 
         dataset_info = api.dataset.get_info_by_id(dataset_id)
@@ -178,12 +198,13 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
 
         for scene in nuscenes.scene:
             current_dataset_id = scene_name_to_dataset[scene["name"]].id
+            self._current_ds_id = current_dataset_id
 
             log = nuscenes.get("log", scene["log_token"])
             sample_token = scene["first_sample_token"]
 
             # * Extract scene's samples
-            scene_samples: List[helpers.Sample] = []
+            scene_samples: Dict[str, helpers.Sample] = {}
             for i in range(scene["nbr_samples"]):
                 sample = nuscenes.get("sample", sample_token)
                 lidar_path, boxes, _ = nuscenes.get_sample_data(sample["data"]["LIDAR_TOP"])
@@ -203,10 +224,12 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
                         nuscenes.get("attribute", attr)["name"] for attr in ann["attribute_tokens"]
                     ]
                     visibility = nuscenes.get("visibility", ann["visibility_token"])["level"]
+                    ann_token = ann["token"]
 
                     ann = helpers.AnnotationObject(
                         name=name,
                         bbox=box,
+                        token=ann_token,
                         instance_token=current_instance_token,
                         parent_token=parent_token,
                         category=category,
@@ -227,12 +250,15 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
                     for sensor, token in sample["data"].items()
                     if sensor.startswith("CAM")
                 ]
-                scene_samples.append(helpers.Sample(timestamp, lidar_path, anns, camera_data))
+                sample_token = sample["token"]
+                scene_samples[sample_token] = helpers.Sample(
+                    timestamp, lidar_path, anns, camera_data
+                )
                 sample_token = sample["next"]
 
             # * Convert and upload pointclouds
             frame_to_pointcloud_ids = {}
-            for idx, sample in enumerate(scene_samples):
+            for idx, sample in enumerate(scene_samples.values()):
                 pcd_path = sample.convert_lidar_to_supervisely()
 
                 pcd_name = fs.get_file_name_with_ext(pcd_path)
@@ -275,9 +301,10 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
 
             # * Convert and upload annotations
             pcd_ann = self.to_supervisely(scene_samples, meta, renamed_classes, renamed_tags)
+
             try:
                 api.pointcloud_episode.annotation.append(
-                    current_dataset_id, pcd_ann, frame_to_pointcloud_ids
+                    current_dataset_id, pcd_ann, frame_to_pointcloud_ids, key_id_map=key_id_map
                 )
                 logger.info(f"Dataset ID:{current_dataset_id} has been successfully uploaded.")
             except Exception as e:
@@ -285,6 +312,15 @@ class NuscenesEpisodesConverter(PointcloudEpisodeConverter):
                 logger.warning(
                     f"Failed to upload annotation for scene: {scene['name']}. Message: {error_msg}"
                 )
+        key_id_map = key_id_map.to_dict()
+        key_id_map.pop("tags")
+        key_id_map.pop("videos")
+        self._custom_data["key_id_map"] = key_id_map
+
+        project_id = dataset_info.project_id
+        current_custom_data = api.project.get_custom_data(project_id)
+        current_custom_data.update(self._custom_data)
+        api.project.update_custom_data(project_id, current_custom_data)
 
         if log_progress:
             if is_development():

supervisely/convert/pointcloud_episodes/nuscenes_conv/nuscenes_helper.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass, field
 from datetime import datetime
 from os import path as osp
 from pathlib import Path
@@ -52,6 +53,7 @@ def trim_description(description: str, max_length: int = 255) -> str:
     return description
 
 
+@dataclass
 class AnnotationObject:
     """
     A class to represent an annotation object in the NuScenes dataset.
@@ -60,6 +62,8 @@ class AnnotationObject:
     :type name: str
     :param bbox: The bounding box coordinates in NuScenes format
     :type bbox: np.ndarray
+    :param token: The unique token identifying the annotation object
+    :type token: str
     :param instance_token: The instance token associated with the annotation object
     :type instance_token: str
     :param parent_token: The token of instance preceding the current object instance
@@ -71,25 +75,14 @@ class AnnotationObject:
     :param visibility: The visibility level of the annotation object
     :type visibility: str
     """
-
-    def __init__(
-        self,
-        name: str,
-        bbox: np.ndarray,
-        instance_token: str,
-        parent_token: str,
-        category: str,
-        attributes: List[str],
-        visibility: str,
-    ):
-        self.name = name
-        self.bbox = bbox
-        self.instance_token = instance_token
-        self.parent_token = parent_token
-
-        self.category = category
-        self.attributes = attributes
-        self.visibility = visibility
+    name: str
+    bbox: np.ndarray
+    token: str
+    instance_token: str
+    parent_token: str
+    category: str
+    attributes: List[str]
+    visibility: str
 
     def to_supervisely(self) -> Cuboid3d:
         box = self.convert_nuscenes_to_BEVBox3D()
@@ -213,38 +206,19 @@ class CamData:
             return (sly_path_img, img_info)
 
 
+@dataclass
 class Sample:
     """
     A class to represent a sample from the NuScenes dataset.
     """
-
-    def __init__(
-        self,
-        timestamp: float,
-        lidar_path: str,
-        anns: List[AnnotationObject],
-        cam_data: List[CamData],
-    ):
-        self._timestamp = datetime.utcfromtimestamp(timestamp / 1e6).isoformat()
-        self._lidar_path = lidar_path
-        self._anns = anns
-        self._cam_data = cam_data
-
-    @property
-    def timestamp(self) -> str:
-        return self._timestamp
-
-    @property
-    def lidar_path(self) -> str:
-        return self._lidar_path
-
-    @property
-    def anns(self) -> List[AnnotationObject]:
-        return self._anns
-
-    @property
-    def cam_data(self) -> List[CamData]:
-        return self._cam_data
+    timestamp_us: float
+    lidar_path: str
+    anns: List[AnnotationObject]
+    cam_data: List[CamData]
+    timestamp: str = field(init=False)
+
+    def __post_init__(self):
+        self.timestamp = datetime.utcfromtimestamp(self.timestamp_us / 1e6).isoformat()
 
     @staticmethod
     def generate_boxes(nuscenes, boxes: List) -> Generator: