supervisely 6.73.320__py3-none-any.whl → 6.73.322__py3-none-any.whl

supervisely/_utils.py

@@ -471,6 +471,39 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
             return loop
 
 
+def sync_call(coro):
+    """
+    This function is used to run asynchronous functions in synchronous context.
+
+    :param coro: Asynchronous function.
+    :type coro: Coroutine
+    :return: Result of the asynchronous function.
+    :rtype: Any
+
+    :Usage example:
+
+    .. code-block:: python
+
+            from supervisely.utils import sync_call
+
+            async def async_function():
+                await asyncio.sleep(1)
+                return "Hello, World!"
+            coro = async_function()
+            result = sync_call(coro)
+            print(result)
+            # Output: Hello, World!
+    """
+
+    loop = get_or_create_event_loop()
+
+    if loop.is_running():
+        future = asyncio.run_coroutine_threadsafe(coro, loop=loop)
+        return future.result()
+    else:
+        return loop.run_until_complete(coro)
+
+
 def get_filename_from_headers(url):
     try:
         response = requests.head(url, allow_redirects=True)

supervisely/api/api.py

@@ -1443,6 +1443,7 @@ class Api:
         chunk_size: int = 8192,
         use_public_api: Optional[bool] = True,
         timeout: httpx._types.TimeoutTypes = 60,
+        **kwargs,
     ) -> AsyncGenerator:
         """
         Performs asynchronous streaming GET or POST request to server with given parameters.
@@ -1486,18 +1487,19 @@ class Api:
         else:
             headers = {**self.headers, **headers}
 
-        if isinstance(data, (bytes, Generator)):
-            content = data
-            json_body = None
-            params = None
-        elif isinstance(data, Dict):
-            json_body = {**data, **self.additional_fields}
-            content = None
-            params = None
+        params = kwargs.get("params", None)
+        if "content" in kwargs or "json_body" in kwargs:
+            content = kwargs.get("content", None)
+            json_body = kwargs.get("json_body", None)
         else:
-            params = data
-            content = None
-            json_body = None
+            if isinstance(data, (bytes, Generator)):
+                content = data
+                json_body = None
+            elif isinstance(data, Dict):
+                json_body = {**data, **self.additional_fields}
+                content = None
+            else:
+                raise ValueError("Data should be either bytes or dict")
 
         if range_start is not None or range_end is not None:
             headers["Range"] = f"bytes={range_start or ''}-{range_end or ''}"
@@ -1512,17 +1514,19 @@ class Api:
                         url,
                         content=content,
                         json=json_body,
-                        params=params,
                         headers=headers,
                         timeout=timeout,
+                        params=params,
                     )
                 elif method_type == "GET":
                     response = self.async_httpx_client.stream(
                         method_type,
                         url,
-                        json=json_body or params,
+                        content=content,
+                        json=json_body,
                         headers=headers,
                         timeout=timeout,
+                        params=params,
                     )
                 else:
                     raise NotImplementedError(

supervisely/api/file_api.py

@@ -33,7 +33,9 @@ from supervisely.io.fs import (
     get_file_name,
     get_file_name_with_ext,
     get_file_size,
+    get_or_create_event_loop,
     list_files_recursively,
+    list_files_recursively_async,
     silent_remove,
 )
 from supervisely.io.fs_cache import FileCache
@@ -2041,7 +2043,7 @@ class FileApi(ModuleApiBase):
         # check_hash: bool = True, #TODO add with resumaple api
         progress_cb: Optional[Union[tqdm, Callable]] = None,
         progress_cb_type: Literal["number", "size"] = "size",
-    ) -> httpx.Response:
+    ) -> None:
         """
         Upload file from local path to Team Files asynchronously.
 
@@ -2057,8 +2059,8 @@ class FileApi(ModuleApiBase):
         :type progress_cb: tqdm or callable, optional
         :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
         :type progress_cb_type: Literal["number", "size"], optional
-        :return: Response from API.
-        :rtype: :class:`httpx.Response`
+        :return: None
+        :rtype: :class:`NoneType`
         :Usage example:
 
             .. code-block:: python
@@ -2087,17 +2089,30 @@ class FileApi(ModuleApiBase):
         }
         if semaphore is None:
             semaphore = self._api.get_default_semaphore()
+        logger.debug(f"Uploading with async to: {dst}. Semaphore: {semaphore}")
         async with semaphore:
             async with aiofiles.open(src, "rb") as fd:
-                item = await fd.read()
-                response = await self._api.post_async(
-                    api_method, content=item, params=json_body, headers=headers
-                )
-                if progress_cb is not None and progress_cb_type == "size":
-                    progress_cb(len(item))
+
+                async def file_chunk_generator():
+                    while True:
+                        chunk = await fd.read(8 * 1024 * 1024)
+                        if not chunk:
+                            break
+                        if progress_cb is not None and progress_cb_type == "size":
+                            progress_cb(len(chunk))
+                        yield chunk
+
+                async for chunk, _ in self._api.stream_async(
+                    method=api_method,
+                    method_type="POST",
+                    data=file_chunk_generator(),  # added as required, but not used inside
+                    headers=headers,
+                    content=file_chunk_generator(),  # used instead of data inside stream_async
+                    params=json_body,
+                ):
+                    pass
                 if progress_cb is not None and progress_cb_type == "number":
                     progress_cb(1)
-                return response
 
     async def upload_bulk_async(
         self,
@@ -2109,6 +2124,7 @@ class FileApi(ModuleApiBase):
         # check_hash: bool = True, #TODO add with resumaple api
         progress_cb: Optional[Union[tqdm, Callable]] = None,
         progress_cb_type: Literal["number", "size"] = "size",
+        enable_fallback: Optional[bool] = True,
     ) -> None:
         """
         Upload multiple files from local paths to Team Files asynchronously.
@@ -2125,6 +2141,8 @@ class FileApi(ModuleApiBase):
         :type progress_cb: tqdm or callable, optional
         :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
         :type progress_cb_type: Literal["number", "size"], optional
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
         :return: None
         :rtype: :class:`NoneType`
         :Usage example:
@@ -2153,19 +2171,134 @@ class FileApi(ModuleApiBase):
                     api.file.upload_bulk_async(8, paths_to_files, paths_to_save)
                 )
         """
-        if semaphore is None:
-            semaphore = self._api.get_default_semaphore()
-        tasks = []
-        for s, d in zip(src_paths, dst_paths):
-            task = self.upload_async(
-                team_id,
-                s,
-                d,
-                semaphore=semaphore,
-                # chunk_size=chunk_size, #TODO add with resumaple api
-                # check_hash=check_hash, #TODO add with resumaple api
-                progress_cb=progress_cb,
-                progress_cb_type=progress_cb_type,
+        try:
+            if semaphore is None:
+                semaphore = self._api.get_default_semaphore()
+            tasks = []
+            for src, dst in zip(src_paths, dst_paths):
+                task = asyncio.create_task(
+                    self.upload_async(
+                        team_id=team_id,
+                        src=src,
+                        dst=dst,
+                        semaphore=semaphore,
+                        # chunk_size=chunk_size, #TODO add with resumaple api
+                        # check_hash=check_hash, #TODO add with resumaple api
+                        progress_cb=progress_cb,
+                        progress_cb_type=progress_cb_type,
+                    )
+                )
+                tasks.append(task)
+            for task in tasks:
+                await task
+        except Exception as e:
+            if enable_fallback:
+                logger.warning(
+                    f"Upload files bulk asynchronously failed. Fallback to synchronous upload.",
+                    exc_info=True,
+                )
+                if progress_cb is not None and progress_cb_type == "number":
+                    logger.warning(
+                        "Progress callback type 'number' is not supported for synchronous upload. "
+                        "Progress callback will be disabled."
+                    )
+                    progress_cb = None
+                self.upload_bulk(
+                    team_id=team_id,
+                    src_paths=src_paths,
+                    dst_paths=dst_paths,
+                    progress_cb=progress_cb,
+                )
+            else:
+                raise e
+
+    async def upload_directory_async(
+        self,
+        team_id: int,
+        local_dir: str,
+        remote_dir: str,
+        change_name_if_conflict: Optional[bool] = True,
+        progress_size_cb: Optional[Union[tqdm, Callable]] = None,
+        replace_if_conflict: Optional[bool] = False,
+        enable_fallback: Optional[bool] = True,
+    ) -> str:
+        """
+        Upload Directory to Team Files from local path.
+        Files are uploaded asynchronously.
+
+        :param team_id: Team ID in Supervisely.
+        :type team_id: int
+        :param local_dir: Path to local Directory.
+        :type local_dir: str
+        :param remote_dir: Path to Directory in Team Files.
+        :type remote_dir: str
+        :param change_name_if_conflict: Checks if given name already exists and adds suffix to the end of the name.
+        :type change_name_if_conflict: bool, optional
+        :param progress_size_cb: Function for tracking download progress.
+        :type progress_size_cb: Progress, optional
+        :param replace_if_conflict: If True, replace existing dir.
+        :type replace_if_conflict: bool, optional
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
+        :return: Path to Directory in Team Files
+        :rtype: :class:`str`
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            path_to_dir = "/My_App_Test/ds1"
+            local_path = "/home/admin/Downloads/My_local_test"
+
+            api.file.upload_directory(9, local_path, path_to_dir)
+        """
+        try:
+            if not remote_dir.startswith("/"):
+                remote_dir = "/" + remote_dir
+
+            if self.dir_exists(team_id, remote_dir):
+                if change_name_if_conflict is True:
+                    res_remote_dir = self.get_free_dir_name(team_id, remote_dir)
+                elif replace_if_conflict is True:
+                    res_remote_dir = remote_dir
+                else:
+                    raise FileExistsError(
+                        f"Directory {remote_dir} already exists in your team (id={team_id})"
+                    )
+            else:
+                res_remote_dir = remote_dir
+
+            local_files = await list_files_recursively_async(local_dir)
+            dir_prefix = local_dir.rstrip("/") + "/"
+            remote_files = [
+                res_remote_dir.rstrip("/") + "/" + file[len(dir_prefix) :] for file in local_files
+            ]
+
+            await self.upload_bulk_async(
+                team_id=team_id,
+                src_paths=local_files,
+                dst_paths=remote_files,
+                progress_cb=progress_size_cb,
             )
-            tasks.append(task)
-        await asyncio.gather(*tasks)
+        except Exception as e:
+            if enable_fallback:
+                logger.warning(
+                    f"Upload directory asynchronously failed. Fallback to synchronous upload.",
+                    exc_info=True,
+                )
+                res_remote_dir = self.upload_directory(
+                    team_id=team_id,
+                    local_dir=local_dir,
+                    remote_dir=res_remote_dir,
+                    change_name_if_conflict=change_name_if_conflict,
+                    progress_size_cb=progress_size_cb,
+                    replace_if_conflict=replace_if_conflict,
+                )
+            else:
+                raise e
+        return res_remote_dir

supervisely/convert/base_converter.py

@@ -65,6 +65,7 @@ class AvailablePointcloudEpisodesConverters:
     SLY = "supervisely"
     BAG = "rosbag"
     LYFT = "lyft"
+    KITTI360 = "kitti360"
 
 
 class AvailableVolumeConverters:

supervisely/convert/pointcloud_episodes/__init__.py

@@ -7,3 +7,4 @@ from supervisely.convert.pointcloud_episodes.lyft.lyft_converter import LyftEpis
 from supervisely.convert.pointcloud_episodes.nuscenes_conv.nuscenes_converter import (
     NuscenesEpisodesConverter,
 )
+from supervisely.convert.pointcloud_episodes.kitti_360.kitti_360_converter import KITTI360Converter

supervisely/convert/pointcloud_episodes/kitti_360/kitti_360_converter.py

@@ -0,0 +1,242 @@
+import os
+from pathlib import Path
+from typing import Optional, List
+from supervisely import PointcloudEpisodeAnnotation, ProjectMeta, is_development, logger, ObjClass, ObjClassCollection
+from supervisely.geometry.cuboid_3d import Cuboid3d
+from supervisely.api.api import Api, ApiField
+from supervisely.convert.base_converter import AvailablePointcloudEpisodesConverters
+from supervisely.convert.pointcloud_episodes.kitti_360.kitti_360_helper import *
+from supervisely.convert.pointcloud_episodes.pointcloud_episodes_converter import PointcloudEpisodeConverter
+from supervisely.io.fs import (
+    file_exists,
+    get_file_name,
+    get_file_name_with_ext,
+    list_files_recursively,
+    silent_remove,
+)
+from supervisely.pointcloud_annotation.pointcloud_episode_frame_collection import PointcloudEpisodeFrameCollection
+from supervisely.pointcloud_annotation.pointcloud_episode_object_collection import PointcloudEpisodeObjectCollection
+from supervisely.pointcloud_annotation.pointcloud_episode_object import PointcloudEpisodeObject
+from supervisely.pointcloud_annotation.pointcloud_episode_frame import PointcloudEpisodeFrame
+from supervisely.pointcloud_annotation.pointcloud_figure import PointcloudFigure
+
+class KITTI360Converter(PointcloudEpisodeConverter):
+
+    class Item:
+
+        def __init__(
+            self,
+            scene_name: str,
+            frame_paths: List[str],
+            ann_data: Annotation3D,
+            poses_path: str,
+            related_images: Optional[tuple] = None,
+            custom_data: Optional[dict] = None,
+        ):
+            self._scene_name = scene_name
+            self._frame_paths = frame_paths
+            self._ann_data = ann_data
+            self._poses_path = poses_path
+            self._related_images = related_images or []
+
+            self._type = "point_cloud_episode"
+            self._custom_data = custom_data if custom_data is not None else {}
+
+    def __init__(self, *args, **kwargs):
+        self._calib_path = None
+        super().__init__(*args, **kwargs)
+
+    def __str__(self) -> str:
+        return AvailablePointcloudEpisodesConverters.KITTI360
+
+    @property
+    def key_file_ext(self) -> str:
+        return ".bin"
+
+    def validate_format(self) -> bool:
+        try:
+            import kitti360scripts
+        except ImportError:
+            logger.warn("Please run 'pip install kitti360Scripts' to import KITTI-360 data.")
+            return False
+
+        self._items = []
+        subdirs = os.listdir(self._input_data)
+        if len(subdirs) == 1:
+            self._input_data = os.path.join(self._input_data, subdirs[0])
+
+        # * Get calibration path
+        calib_dir = next(iter([(Path(path).parent).as_posix() for path in list_files_recursively(self._input_data, [".txt"], None, True) if Path(path).stem.startswith("calib")]), None)
+        if calib_dir is None:
+            return False
+        self._calib_path = calib_dir
+
+        # * Get pointcloud files paths
+        velodyne_files = list_files_recursively(self._input_data, [".bin"], None, True)
+        if len(velodyne_files) == 0:
+            return False
+
+        # * Get annotation files paths and related images
+        boxes_ann_files = list_files_recursively(self._input_data, [".xml"], None, True)
+        if len(boxes_ann_files) == 0:
+            return False
+        rimage_files = list_files_recursively(self._input_data, [".png"], None, True)
+
+        kitti_anns = []
+        for ann_file in boxes_ann_files:
+            key_name = Path(ann_file).stem
+
+            # * Get pointcloud files
+            frame_paths = []
+            for path in velodyne_files:
+                if key_name in Path(path).parts:
+                    frame_paths.append(path)
+            if len(frame_paths) == 0:
+                logger.warn("No frames found for name: %s", key_name)
+                continue
+
+            # * Get related images
+            rimages = []
+            for rimage in rimage_files:
+                path = Path(rimage)
+                if key_name in path.parts:
+                    cam_name = path.parts[-3]
+                    rimages.append((cam_name, rimage))
+
+            # * Get poses
+            poses_filter = (
+                lambda x: x.endswith("cam0_to_world.txt") and key_name in Path(x).parts
+            )
+            poses_path = next(
+                path
+                for path in list_files_recursively(self._input_data, [".txt"], None, True)
+                if poses_filter(path)
+            )
+            if poses_path is None:
+                logger.warn("No poses found for name: %s", key_name)
+                continue
+
+            # * Parse annotation
+            ann = Annotation3D(ann_file)
+            kitti_anns.append(ann)
+
+            self._items.append(
+                self.Item(key_name, frame_paths, ann, poses_path, rimages)
+            )
+
+        # * Get object class names for meta
+        obj_class_names = set()
+        for ann in kitti_anns:
+            for obj in ann.get_objects():
+                obj_class_names.add(obj.name)
+        obj_classes = [ObjClass(obj_class, Cuboid3d) for obj_class in obj_class_names]
+        self._meta = ProjectMeta(obj_classes=ObjClassCollection(obj_classes))
+        return self.items_count > 0
+
+    def to_supervisely(
+        self,
+        item,
+        meta: ProjectMeta,
+        renamed_classes: dict = {},
+        renamed_tags: dict = {},
+        static_transformations: StaticTransformations = None,
+    ) -> PointcloudEpisodeAnnotation:
+        static_transformations.set_cam2world(item._poses_path)
+
+        frame_cnt = len(item._frame_paths)
+        objs, frames = [], []
+
+        frame_idx_to_figures = {idx: [] for idx in range(frame_cnt)}
+        for obj in item._ann_data.get_objects():
+            pcd_obj = PointcloudEpisodeObject(meta.get_obj_class(obj.name))
+            objs.append(pcd_obj)
+
+            for idx in range(frame_cnt):
+                if obj.start_frame <= idx <= obj.end_frame:
+                    tr_matrix = static_transformations.world_to_velo_transformation(obj, idx)
+                    geom = convert_kitti_cuboid_to_supervisely_geometry(tr_matrix)
+                    frame_idx_to_figures[idx].append(PointcloudFigure(pcd_obj, geom, idx))
+        for idx, figures in frame_idx_to_figures.items():
+            frame = PointcloudEpisodeFrame(idx, figures)
+            frames.append(frame)
+        obj_collection = PointcloudEpisodeObjectCollection(objs)
+        frame_collection = PointcloudEpisodeFrameCollection(frames)
+        return PointcloudEpisodeAnnotation(
+            frame_cnt, objects=obj_collection, frames=frame_collection
+        )
+
+    def upload_dataset(self, api: Api, dataset_id: int, batch_size: int = 1, log_progress=True):
+        meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
+
+        dataset_info = api.dataset.get_info_by_id(dataset_id)
+        if log_progress:
+            progress, progress_cb = self.get_progress(sum([len(item._frame_paths) for item in self._items]), "Converting pointcloud episodes...")
+        else:
+            progress_cb = None
+        static_transformations = StaticTransformations(self._calib_path)
+        scene_ds = dataset_info
+        multiple_items = self.items_count > 1
+        for item in self._items:
+            scene_ds = api.dataset.create(dataset_info.project_id, item._scene_name, parent_id=dataset_id) if multiple_items else dataset_info
+            frame_to_pcd_ids = {}
+            for idx, frame_path in enumerate(item._frame_paths):
+                # * Convert pointcloud from ".bin" to ".pcd"
+                pcd_path = str(Path(frame_path).with_suffix(".pcd"))
+                if file_exists(pcd_path):
+                    logger.warning(f"Overwriting file with path: {pcd_path}")
+                convert_bin_to_pcd(frame_path, pcd_path)
+
+                # * Upload pointcloud
+                pcd_name = get_file_name_with_ext(pcd_path)
+                info = api.pointcloud_episode.upload_path(scene_ds.id, pcd_name, pcd_path, {"frame": idx})
+                pcd_id = info.id
+                frame_to_pcd_ids[idx] = pcd_id
+
+                # * Clean up
+                silent_remove(pcd_path)
+
+                if log_progress:
+                    progress_cb(1)
+
+            # * Upload photocontext
+            rimage_jsons = []
+            cam_names = []
+            hashes = api.pointcloud_episode.upload_related_images(
+                [rimage_path for _, rimage_path in item._related_images]
+            )
+            for (cam_name, rimage_path), img, pcd_id in zip(
+                item._related_images, hashes, list(frame_to_pcd_ids.values())
+            ):
+                cam_num = int(cam_name[-1])
+                rimage_info = convert_calib_to_image_meta(
+                    get_file_name(rimage_path), static_transformations, cam_num
+                )
+                image_json = {
+                    ApiField.ENTITY_ID: pcd_id,
+                    ApiField.NAME: cam_name,
+                    ApiField.HASH: img,
+                    ApiField.META: rimage_info[ApiField.META],
+                }
+                rimage_jsons.append(image_json)
+                cam_names.append(cam_name)
+            if rimage_jsons:
+                api.pointcloud_episode.add_related_images(rimage_jsons, cam_names)
+
+            # * Convert annotation and upload
+            try:
+                ann = self.to_supervisely(
+                    item, meta, renamed_classes, renamed_tags, static_transformations
+                )
+                api.pointcloud_episode.annotation.append(scene_ds.id, ann, frame_to_pcd_ids)
+            except Exception as e:
+                logger.error(
+                    f"Failed to upload annotation for scene: {scene_ds.name}. Error: {repr(e)}",
+                    stack_info=False,
+                )
+                continue
+
+            logger.info(f"Dataset ID:{scene_ds.id} has been successfully uploaded.")
+
+        if log_progress:
+            if is_development():
+                progress.close()