supervisely 6.73.220__py3-none-any.whl → 6.73.222__py3-none-any.whl

supervisely/api/volume/volume_api.py

@@ -1,6 +1,8 @@
+import asyncio
 import os
-from typing import Callable, List, NamedTuple, Optional, Union
+from typing import AsyncGenerator, Callable, List, NamedTuple, Optional, Union
 
+import aiofiles
 from tqdm import tqdm
 from tqdm.contrib.logging import logging_redirect_tqdm
 
@@ -19,6 +21,7 @@ from supervisely.io.fs import (
     ensure_base_path,
     get_bytes_hash,
     get_file_ext,
+    get_file_hash_async,
     get_file_name,
     get_file_name_with_ext,
 )
@@ -852,7 +855,7 @@ class VolumeApi(RemoveableBulkModuleApi):
 
     def _download(self, id: int, is_stream: bool = False):
         """
-        Private method for volume volume downloading.
+        Private method for volume downloading.
 
         :param id: Volume ID in Supervisely.
         :type id: int
@@ -1236,3 +1239,221 @@ class VolumeApi(RemoveableBulkModuleApi):
                 )
             )
         return volume_infos
+
+    async def _download_async(
+        self,
+        id: int,
+        is_stream: bool = False,
+        range_start: Optional[int] = None,
+        range_end: Optional[int] = None,
+        headers: Optional[dict] = None,
+        chunk_size: int = 1024 * 1024,
+    ) -> AsyncGenerator:
+        """
+        Download Volume with given ID asynchronously.
+        If is_stream is True, returns stream of bytes, otherwise returns response object.
+        For streaming, returns tuple of chunk and hash.
+
+        :param id: Volume ID in Supervisely.
+        :type id: int
+        :param is_stream: If True, returns stream of bytes, otherwise returns response object.
+        :type is_stream: bool, optional
+        :param range_start: Start byte of range for partial download.
+        :type range_start: int, optional
+        :param range_end: End byte of range for partial download.
+        :type range_end: int, optional
+        :param headers: Headers for request.
+        :type headers: dict, optional
+        :param chunk_size: Size of chunk for downloading. Default is 1MB.
+        :type chunk_size: int, optional
+        :return: Stream of bytes or response object.
+        :rtype: AsyncGenerator
+        """
+        api_method_name = "volumes.download"
+
+        json_body = {ApiField.ID: id}
+
+        if is_stream:
+            async for chunk, hhash in self._api.stream_async(
+                api_method_name,
+                "POST",
+                json_body,
+                headers=headers,
+                range_start=range_start,
+                range_end=range_end,
+                chunk_size=chunk_size,
+            ):
+                yield chunk, hhash
+        else:
+            response = await self._api.post_async(api_method_name, json_body, headers=headers)
+            yield response
+
+    async def download_path_async(
+        self,
+        id: int,
+        path: str,
+        semaphore: Optional[asyncio.Semaphore] = None,
+        range_start: Optional[int] = None,
+        range_end: Optional[int] = None,
+        headers: Optional[dict] = None,
+        chunk_size: int = 1024 * 1024,
+        check_hash: bool = True,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "number",
+    ) -> None:
+        """
+        Downloads Volume with given ID to local path.
+
+        :param id: Volume ID in Supervisely.
+        :type id: int
+        :param path: Local save path for Volume.
+        :type path: str
+        :param semaphore: Semaphore for limiting the number of simultaneous downloads.
+        :type semaphore: :class:`asyncio.Semaphore`, optional
+        :param range_start: Start byte of range for partial download.
+        :type range_start: int, optional
+        :param range_end: End byte of range for partial download.
+        :type range_end: int, optional
+        :param headers: Headers for request.
+        :type headers: dict, optional
+        :param chunk_size: Size of chunk for downloading. Default is 1MB.
+        :type chunk_size: int, optional
+        :param check_hash: If True, checks hash of downloaded file.
+                        Check is not supported for partial downloads.
+                        When range is set, hash check is disabled.
+        :type check_hash: bool, optional
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: tqdm or callable, optional
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "number".
+        :type progress_cb_type: str, optional
+        :return: None
+        :rtype: :class:`NoneType`
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+            import asyncio
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            volume_info = api.volume.get_info_by_id(770918)
+            save_path = os.path.join("/path/to/save/", volume_info.name)
+
+            semaphore = asyncio.Semaphore(100)
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            loop.run_until_complete(
+                        api.volume.download_path_async(volume_info.id, save_path, semaphore)
+                )
+        """
+
+        if range_start is not None or range_end is not None:
+            check_hash = False  # Hash check is not supported for partial downloads
+            headers = headers or {}
+            headers["Range"] = f"bytes={range_start or ''}-{range_end or ''}"
+            logger.debug(f"Image ID: {id}. Setting Range header: {headers['Range']}")
+
+        writing_method = "ab" if range_start not in [0, None] else "wb"
+
+        ensure_base_path(path)
+        hash_to_check = None
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+        async with semaphore:
+            async with aiofiles.open(path, writing_method) as fd:
+                async for chunk, hhash in self._download_async(
+                    id,
+                    is_stream=True,
+                    headers=headers,
+                    range_start=range_start,
+                    range_end=range_end,
+                    chunk_size=chunk_size,
+                ):
+                    await fd.write(chunk)
+                    hash_to_check = hhash
+                    if progress_cb is not None and progress_cb_type == "size":
+                        progress_cb(len(chunk))
+            if check_hash:
+                if hash_to_check is not None:
+                    downloaded_file_hash = await get_file_hash_async(path)
+                    if hash_to_check != downloaded_file_hash:
+                        raise RuntimeError(
+                            f"Downloaded hash of volume with ID:{id} does not match the expected hash: {downloaded_file_hash} != {hash_to_check}"
+                        )
+            if progress_cb is not None and progress_cb_type == "number":
+                progress_cb(1)
+
+    async def download_paths_async(
+        self,
+        ids: List[int],
+        paths: List[str],
+        semaphore: Optional[asyncio.Semaphore] = None,
+        headers: Optional[dict] = None,
+        chunk_size: int = 1024 * 1024,
+        check_hash: bool = True,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "number",
+    ) -> None:
+        """
+        Download Volumes with given IDs and saves them to given local paths asynchronously.
+
+        :param ids: List of Volume IDs in Supervisely.
+        :type ids: :class:`List[int]`
+        :param paths: Local save paths for Volumes.
+        :type paths: :class:`List[str]`
+        :param semaphore: Semaphore for limiting the number of simultaneous downloads.
+        :type semaphore: :class:`asyncio.Semaphore`, optional
+        :param headers: Headers for request.
+        :type headers: dict, optional
+        :param chunk_size: Size of chunk for downloading. Default is 1MB.
+        :type chunk_size: int, optional
+        :param check_hash: If True, checks hash of downloaded file.
+        :type check_hash: bool, optional
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: tqdm or callable, optional
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "number".
+        :type progress_cb_type: str, optional
+        :raises: :class:`ValueError` if len(ids) != len(paths)
+        :return: None
+        :rtype: :class:`NoneType`
+
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+            import asyncio
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            ids = [770914, 770915]
+            paths = ["/path/to/save/volume1.nrrd", "/path/to/save/volume2.nrrd"]
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            loop.run_until_complete(api.volume.download_paths_async(ids, paths))
+        """
+        if len(ids) == 0:
+            return
+        if len(ids) != len(paths):
+            raise ValueError(f'Can not match "ids" and "paths" lists, {len(ids)} != {len(paths)}')
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+        tasks = []
+        for img_id, img_path in zip(ids, paths):
+            task = self.download_path_async(
+                img_id,
+                img_path,
+                semaphore,
+                headers=headers,
+                chunk_size=chunk_size,
+                check_hash=check_hash,
+                progress_cb=progress_cb,
+                progress_cb_type=progress_cb_type,
+            )
+            tasks.append(task)
+        await asyncio.gather(*tasks)

supervisely/app/development/__init__.py

@@ -1,4 +1,5 @@
 from supervisely.app.development.development import (
     supervisely_vpn_network,
     create_debug_task,
+    enable_advanced_debug,
 )

supervisely/app/development/development.py

@@ -6,6 +6,8 @@ from typing import Any, Dict, Optional
 
 import requests
 
+import supervisely.io.env as sly_env
+from supervisely._utils import is_development
 from supervisely.api.api import Api
 from supervisely.io.fs import mkdir
 from supervisely.sly_logger import logger
@@ -154,11 +156,12 @@ def supervisely_vpn_network(
 
 
 def create_debug_task(
-    team_id: int, port: int = 8000, update_status: bool = True
+    team_id: int = None, port: int = 8000, update_status: bool = True
 ) -> Dict[str, Any]:
     """Gets or creates a debug task for the current user.
 
-    :param team_id: The ID of the team to create the task in.
+    :param team_id: The ID of the team to create the task in, if not provided, the function
+        will try to obtain it from environment variables. Default is None.
     :type team_id: int
     :param port: The port to redirect the requests to. Default is 8000.
     :type port: int
@@ -167,6 +170,14 @@ def create_debug_task(
     :return: The task details.
     :rtype: Dict[str, Any]
     """
+    team_id = team_id or sly_env.team_id()
+    if not team_id:
+        raise ValueError(
+            "Team ID is not provided and cannot be obtained from environment variables."
+            "The debug task cannot be created. Please provide the team ID as an argument "
+            "or set the TEAM_ID environment variable."
+        )
+
     api = Api()
     me = api.user.get_my_info()
     session_name = me.login + "-development"
@@ -200,4 +211,87 @@ def create_debug_task(
     if update_status:
         logger.info(f"Task status will be updated to STARTED for task with ID: {task['id']}")
         api.task.update_status(task["id"], api.task.Status.STARTED)
+
     return task
+
+
+def enable_advanced_debug(
+    team_id: int = None,
+    port: int = 8000,
+    update_status: bool = True,
+    vpn_action: Literal["up", "down"] = "up",
+    vpn_raise_on_error: bool = True,
+    only_for_development: bool = True,
+) -> Optional[int]:
+    """Enables advanced debugging for the app.
+    At first, it establishes a WireGuard VPN connection to the Supervisely network.
+    And then creates a debug task to redirect requests to the local machine.
+
+    Please, ensure that the Team ID was provided, or set as TEAM_ID environment variable.
+    All other parameters can be omitted if using the default instance settings.
+
+    :param team_id: The ID of the team to create the task in, if not provided, the function
+        will try to obtain it from environment variables. Default is None.
+    :type team_id: int
+    :param port: The port to redirect the requests to. Default is 8000.
+    :type port: int
+    :param update_status: If True, the task status will be updated to STARTED.
+    :type update_status: bool
+    :param vpn_action: The action to perform with the VPN connection, either "up" or "down". Default is "up".
+    :type vpn_action: Literal["up", "down"]
+    :param vpn_raise_on_error: If True, an exception will be raised if an error occurs while connecting to VPN.
+    :type vpn_raise_on_error: bool
+    :param only_for_development: If True, the debugging will be started only if the app is running in development mode.
+        It's not recommended to set this parameter to False in production environments.
+    :type only_for_development: bool
+    :return: The task ID of the debug task or None if the debugging was not started.
+    :rtype: Optional[int]
+
+    :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            # Ensure that the TEAM_ID environment variable is set.
+            # Or provide the team ID as an argument to the function.
+            os.environ['TEAM_ID'] = 123456
+
+            # The task ID can be used to make requests to the app.
+            task_id = sly.app.development.enable_advanced_debug()
+
+            # An example of how to send a request to the app using the task ID.
+            data = {"project_id": 789, "force": True}
+            api.task.send_request(task_id, "endpoint-name-here", data, skip_response=True)
+    """
+
+    if only_for_development and not is_development():
+        logger.debug(
+            "Advanced debugging was not started because the app is not running in development mode. "
+            "If you need to force the debugging, set the only_for_development argument to False. "
+            "Use this parameter with caution, and do not set to False in production environments."
+        )
+        return None
+
+    logger.debug(
+        "Starting advanced debugging, will create a wireguard VPN connection and create "
+        "or use an existing debug task to redirect requests to the local machine. "
+        "Learn more about this feature in Supervisely Developer Portal: "
+        "https://developer.supervisely.com/app-development/advanced/advanced-debugging"
+    )
+
+    supervisely_vpn_network(action=vpn_action, raise_on_error=vpn_raise_on_error)
+    task = create_debug_task(team_id=team_id, port=port, update_status=update_status)
+    task_id = task.get("id", None)
+
+    logger.debug(
+        f"Advanced debugging has been started. "
+        f"VPN connection has been established and debug task has been create. Task ID: {task_id}. "
+        "The metod will return the task ID, you can use it to make requests to the app."
+    )
+
+    return task_id

supervisely/app/fastapi/subapp.py

@@ -1,4 +1,5 @@
 import os
+import inspect
 import signal
 import sys
 from contextlib import suppress
@@ -1012,11 +1013,15 @@ class Application(metaclass=Singleton):
             self._graceful_stop_event.set()
         return suppress(self.StopException)
 
-    def event(self, event: Event) -> Callable:
+    def event(self, event: Event, use_state: bool = False) -> Callable:
         """Decorator to register posts to specific endpoints.
+        Supports both async and sync functions.
 
         :param event: event to register (e.g. `Event.Brush.LeftMouseReleased`)
         :type event: Event
+        :param use_state: if set to True, data will be extracted from request.state.state,
+            otherwise from request.state.context, defaults to False
+        :type use_state: bool, optional
         :return: decorator
         :rtype: Callable
 
@@ -1037,9 +1042,19 @@ class Application(metaclass=Singleton):
         def inner(func: Callable) -> Callable:
             server = self.get_server()
 
-            @server.post(event.endpoint)
-            def wrapper(request: Request):
-                return func(request.state.api, event.from_json(request.state.context))
+            if inspect.iscoroutinefunction(func):
+
+                @server.post(event.endpoint)
+                async def wrapper(request: Request):
+                    data = request.state.state if use_state else request.state.context
+                    return await func(request.state.api, event.from_json(data))
+
+            else:
+
+                @server.post(event.endpoint)
+                def wrapper(request: Request):
+                    data = request.state.state if use_state else request.state.context
+                    return func(request.state.api, event.from_json(data))
 
             return wrapper
 

supervisely/convert/base_converter.py

@@ -5,11 +5,12 @@ from typing import Dict, List, Optional, Tuple, Union
 
 from tqdm import tqdm
 
-from supervisely._utils import is_production
+from supervisely._utils import batched, is_production
 from supervisely.annotation.annotation import Annotation
 from supervisely.annotation.tag_meta import TagValueType
 from supervisely.api.api import Api
-from supervisely.io.fs import get_file_ext, get_file_name_with_ext
+from supervisely.io.env import team_id
+from supervisely.io.fs import get_file_ext, get_file_name_with_ext, silent_remove
 from supervisely.project.project_meta import ProjectMeta
 from supervisely.project.project_settings import LabelingInterface
 from supervisely.sly_logger import logger
@@ -145,12 +146,16 @@ class BaseConverter:
         remote_files_map: Optional[Dict[str, str]] = None,
     ):
         self._input_data: str = input_data
-        self._items: List[self.BaseItem] = []
+        self._items: List[BaseConverter.BaseItem] = []
         self._meta: ProjectMeta = None
         self._labeling_interface = labeling_interface or LabelingInterface.DEFAULT
+
+        # import as links settings
         self._upload_as_links: bool = upload_as_links
         self._remote_files_map: Optional[Dict[str, str]] = remote_files_map
         self._supports_links = False  # if converter supports uploading by links
+        self._api = Api.from_env() if self._upload_as_links else None
+        self._team_id = team_id() if self._upload_as_links else None
         self._converter = None
 
         if self._labeling_interface not in LabelingInterface.values():
@@ -286,15 +291,23 @@ class BaseConverter:
             return found_formats[0]
 
     def _collect_items_if_format_not_detected(self):
+        from supervisely.convert.pointcloud_episodes.pointcloud_episodes_converter import (
+            PointcloudEpisodeConverter,
+        )
+
         only_modality_items = True
         unsupported_exts = set()
         items = []
+        is_episode = isinstance(self, PointcloudEpisodeConverter)
         for root, _, files in os.walk(self._input_data):
             for file in files:
                 full_path = os.path.join(root, file)
                 ext = get_file_ext(full_path)
                 if ext.lower() in self.allowed_exts:  # pylint: disable=no-member
-                    items.append(self.Item(full_path))  # pylint: disable=no-member
+                    if is_episode:
+                        items.append(self.Item(full_path, len(items)))  # pylint: disable=no-member
+                    else:
+                        items.append(self.Item(full_path))  # pylint: disable=no-member
                     continue
                 only_modality_items = False
                 if ext.lower() in self.unsupported_exts:
@@ -411,3 +424,39 @@ class BaseConverter:
 
             return meta1.clone(project_settings=new_settings)
         return meta1
+
+    def _download_remote_ann_files(self) -> None:
+        """
+        Download all annotation files from Cloud Storage to the local storage.
+        Needed to detect annotation format if "upload_as_links" is enabled.
+        """
+        if not self.upload_as_links:
+            return
+
+        files_to_download = {
+            l: r for l, r in self._remote_files_map.items() if get_file_ext(l) == self.ann_ext
+        }
+        if not files_to_download:
+            return
+
+        import asyncio
+
+        loop = asyncio.get_event_loop()
+        _, progress_cb = self.get_progress(
+            len(files_to_download),
+            "Downloading annotation files from remote storage",
+        )
+
+        for local_path in files_to_download.keys():
+            silent_remove(local_path)
+
+        logger.info("Downloading annotation files from remote storage...")
+        loop.run_until_complete(
+            self._api.storage.download_bulk_async(
+                team_id=self._team_id,
+                remote_paths=list(files_to_download.values()),
+                local_save_paths=list(files_to_download.keys()),
+                progress_cb=progress_cb,
+            )
+        )
+        logger.info("Annotation files downloaded successfully")

supervisely/convert/converter.py

@@ -1,15 +1,16 @@
 import os
 from pathlib import Path
+from typing import Optional
 
 from tqdm import tqdm
 
-from typing import Literal, Optional
-
 from supervisely._utils import is_production
 from supervisely.api.api import Api
 from supervisely.app import get_data_dir
 from supervisely.convert.image.csv.csv_converter import CSVConverter
-from supervisely.convert.image.high_color.high_color_depth import HighColorDepthImageConverter
+from supervisely.convert.image.high_color.high_color_depth import (
+    HighColorDepthImageConverter,
+)
 from supervisely.convert.image.image_converter import ImageConverter
 from supervisely.convert.pointcloud.pointcloud_converter import PointcloudConverter
 from supervisely.convert.pointcloud_episodes.pointcloud_episodes_converter import (
@@ -28,10 +29,10 @@ from supervisely.io.fs import (
     touch,
     unpack_archive,
 )
+from supervisely.project.project_settings import LabelingInterface
 from supervisely.project.project_type import ProjectType
 from supervisely.sly_logger import logger
 from supervisely.task.progress import Progress
-from supervisely.project.project_settings import LabelingInterface
 
 
 class ImportManager:
@@ -164,7 +165,7 @@ class ImportManager:
         dir_path = remote_path.rstrip("/") if is_dir else os.path.dirname(remote_path)
         dir_name = os.path.basename(dir_path)
 
-        local_path = os.path.join(get_data_dir(), dir_name)
+        local_path = os.path.abspath(os.path.join(get_data_dir(), dir_name))
         mkdir(local_path, remove_content_if_exists=True)
 
         if is_dir:

supervisely/convert/image/image_converter.py

@@ -137,8 +137,8 @@ class ImageConverter(BaseConverter):
                 if item.path is None:
                     continue  # image has failed validation
                 item.name = f"{get_file_name(item.path)}{get_file_ext(item.path).lower()}"
-                if self.upload_as_links:
-                    ann = None  # TODO: implement
+                if self.upload_as_links and not self.supports_links:
+                    ann = None
                 else:
                     ann = self.to_supervisely(item, meta, renamed_classes, renamed_tags)
                 name = generate_free_name(
@@ -160,19 +160,29 @@ class ImageConverter(BaseConverter):
             with ApiContext(
                 api=api, project_id=project_id, dataset_id=dataset_id, project_meta=meta
             ):
-                upload_method = (
-                    api.image.upload_links if self.upload_as_links else api.image.upload_paths
-                )
-                img_infos = upload_method(
-                    dataset_id,
-                    item_names,
-                    item_paths,
-                    metas=item_metas,
-                    conflict_resolution="rename",
-                )
+                if self.upload_as_links:
+                    img_infos = api.image.upload_links(
+                        dataset_id,
+                        item_names,
+                        item_paths,
+                        metas=item_metas,
+                        conflict_resolution="rename",
+                        force_metadata_for_links=False,
+                    )
+                else:
+                    img_infos = api.image.upload_paths(
+                        dataset_id,
+                        item_names,
+                        item_paths,
+                        metas=item_metas,
+                        conflict_resolution="rename",
+                    )
+
                 img_ids = [img_info.id for img_info in img_infos]
                 if len(anns) == len(img_ids):
-                    api.annotation.upload_anns(img_ids, anns)
+                    api.annotation.upload_anns(
+                        img_ids, anns, skip_bounds_validation=self.upload_as_links
+                    )
 
             if log_progress:
                 progress_cb(len(batch))
@@ -188,6 +198,9 @@ class ImageConverter(BaseConverter):
         return image_helper.validate_image(path)
 
     def is_image(self, path: str) -> bool:
+        if self._upload_as_links and self.supports_links:
+            ext = get_file_ext(path)
+            return ext.lower() in self.allowed_exts
         mimetypes.add_type("image/heic", ".heic")  # to extend types_map
         mimetypes.add_type("image/heif", ".heif")  # to extend types_map
         mimetypes.add_type("image/jpeg", ".jfif")  # to extend types_map

supervisely/convert/image/sly/fast_sly_image_converter.py

@@ -20,6 +20,10 @@ from supervisely.convert.image.image_helper import validate_image_bounds
 
 class FastSlyImageConverter(SLYImageConverter, ImageConverter):
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._supports_links = False
+
     def validate_format(self) -> bool:
 
         detected_ann_cnt = 0