supervisely 6.73.247__tar.gz → 6.73.249__tar.gz

{supervisely-6.73.247/supervisely.egg-info → supervisely-6.73.249}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.247
+Version: 6.73.249
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.247 → supervisely-6.73.249}/supervisely/api/annotation_api.py

@@ -1418,13 +1418,14 @@ class AnnotationApi(ModuleApi):
             # if so, download them separately and update the annotation
             if len(additonal_geometries) > 0:
                 figure_ids = list(additonal_geometries.keys())
-                figures = self._api.image.figure.download_geometries_batch(
+                figures = await self._api.image.figure.download_geometries_batch_async(
                     figure_ids,
                     (
                         progress_cb
                         if progress_cb is not None and progress_cb_type == "size"
                         else None
                     ),
+                    semaphore=semaphore,
                 )
                 for figure_id, geometry in zip(figure_ids, figures):
                     label_idx = additonal_geometries[figure_id]
@@ -1515,3 +1516,131 @@ class AnnotationApi(ModuleApi):
             tasks.append(task)
         ann_infos = await asyncio.gather(*tasks)
         return ann_infos
+
+    async def download_bulk_async(
+        self,
+        dataset_id: int,
+        image_ids: List[int],
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        with_custom_data: Optional[bool] = False,
+        force_metadata_for_links: Optional[bool] = True,
+        semaphore: Optional[asyncio.Semaphore] = None,
+    ) -> List[AnnotationInfo]:
+        """
+        Get list of AnnotationInfos for given dataset ID from API.
+        This method is optimized for downloading a large number of small size annotations with a single API call.
+
+        :param dataset_id: Dataset ID in Supervisely.
+        :type dataset_id: int
+        :param image_ids: List of integers.
+        :type image_ids: List[int]
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: tqdm
+        :param with_custom_data: Include custom data in the response.
+        :type with_custom_data: bool, optional
+        :param force_metadata_for_links: Force metadata for links.
+        :type force_metadata_for_links: bool, optional
+        :param semaphore: Semaphore for limiting the number of simultaneous downloads.
+        :type semaphore: asyncio.Semaphore, optional
+        :return: Information about Annotations. See :class:`info_sequence<info_sequence>`
+        :rtype: :class:`List[AnnotationInfo]`
+
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            dataset_id = 254737
+            image_ids = [121236918, 121236919]
+            p = tqdm(desc="Annotations downloaded: ", total=len(image_ids))
+
+            ann_infos = await api.annotation.download_bulk_async(dataset_id, image_ids, progress_cb=p)
+
+            Optimizing the download process by using the context to avoid redundant API calls.:
+            # 1. Download the project meta
+            project_id = api.dataset.get_info_by_id(dataset_id).project_id
+            project_meta = api.project.get_meta(project_id)
+
+            # 2. Use the context to avoid redundant API calls
+            dataset_id = 254737
+            image_ids = [121236918, 121236919]
+            with sly.ApiContext(api, dataset_id=dataset_id, project_id=project_id, project_meta=project_meta):
+                ann_infos = await api.annotation.download_bulk_async(dataset_id, image_ids)
+        """
+        if semaphore is None:
+            semaphore = self._api.get_default_semaphore()
+
+        # use context to avoid redundant API calls
+        context = self._api.optimization_context
+        context_dataset_id = context.get("dataset_id")
+        project_meta = context.get("project_meta")
+        project_id = context.get("project_id")
+        if dataset_id != context_dataset_id:
+            context["dataset_id"] = dataset_id
+            project_id, project_meta = None, None
+
+        if not isinstance(project_meta, ProjectMeta):
+            if project_id is None:
+                project_id = self._api.dataset.get_info_by_id(dataset_id).project_id
+                context["project_id"] = project_id
+            project_meta = ProjectMeta.from_json(self._api.project.get_meta(project_id))
+            context["project_meta"] = project_meta
+
+        need_download_alpha_masks = False
+        for obj_cls in project_meta.obj_classes:
+            if obj_cls.geometry_type == AlphaMask:
+                need_download_alpha_masks = True
+                break
+
+        id_to_ann = {}
+        for batch in batched(image_ids):
+            json_data = {
+                ApiField.DATASET_ID: dataset_id,
+                ApiField.IMAGE_IDS: batch,
+                ApiField.WITH_CUSTOM_DATA: with_custom_data,
+                ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
+                ApiField.INTEGER_COORDS: False,
+            }
+            async with semaphore:
+                results = await self._api.post_async("annotations.bulk.info", json=json_data)
+                results = results.json()
+            if need_download_alpha_masks is True:
+                additonal_geometries = defaultdict(tuple)
+                for ann_idx, ann_dict in enumerate(results):
+                    # check if there are any AlphaMask geometries in the batch
+                    for label_idx, label in enumerate(
+                        ann_dict[ApiField.ANNOTATION][AnnotationJsonFields.LABELS]
+                    ):
+                        if label[LabelJsonFields.GEOMETRY_TYPE] == AlphaMask.geometry_name():
+                            figure_id = label[LabelJsonFields.ID]
+                            additonal_geometries[figure_id] = (ann_idx, label_idx)
+
+                # if there are any AlphaMask geometries, download them separately and update the annotation
+                if len(additonal_geometries) > 0:
+                    figure_ids = list(additonal_geometries.keys())
+                    figures = await self._api.image.figure.download_geometries_batch_async(
+                        figure_ids, semaphore=semaphore
+                    )
+                    for figure_id, geometry in zip(figure_ids, figures):
+                        ann_idx, label_idx = additonal_geometries[figure_id]
+                        results[ann_idx][ApiField.ANNOTATION][AnnotationJsonFields.LABELS][
+                            label_idx
+                        ].update({BITMAP: geometry})
+
+            for ann_dict in results:
+                # Convert annotation to pixel coordinate system
+                ann_dict[ApiField.ANNOTATION] = Annotation._to_pixel_coordinate_system_json(
+                    ann_dict[ApiField.ANNOTATION]
+                )
+                ann_info = self._convert_json_info(ann_dict)
+                id_to_ann[ann_info.image_id] = ann_info
+
+            if progress_cb is not None:
+                progress_cb(len(batch))
+        ordered_results = [id_to_ann[image_id] for image_id in image_ids]
+        return ordered_results

{supervisely-6.73.247 → supervisely-6.73.249}/supervisely/api/api.py

@@ -1437,7 +1437,7 @@ class Api:
         :type range_start: int, optional
         :param range_end: End byte position for streaming.
         :type range_end: int, optional
-        :param chunk_size: Size of the chunk to read from the stream.
+        :param chunk_size: Size of the chunk to read from the stream. Default is 8192.
         :type chunk_size: int, optional
         :param use_public_api: Define if public API should be used.
         :type use_public_api: bool, optional

{supervisely-6.73.247 → supervisely-6.73.249}/supervisely/api/entity_annotation/figure_api.py

@@ -3,10 +3,21 @@
 # docs
 from __future__ import annotations
 
+import asyncio
 import json
 import re
 from collections import defaultdict
-from typing import Callable, Dict, Generator, List, NamedTuple, Optional, Tuple, Union
+from typing import (
+    AsyncGenerator,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    NamedTuple,
+    Optional,
+    Tuple,
+    Union,
+)
 
 import numpy as np
 from requests_toolbelt import MultipartDecoder, MultipartEncoder
@@ -608,3 +619,81 @@ class FigureApi(RemoveableBulkModuleApi):
                 )
             encoder = MultipartEncoder(fields=fields)
             self._api.post("figures.bulk.upload.geometry", encoder)
+
+    async def _download_geometries_generator_async(
+        self, ids: List[int], semaphore: Optional[asyncio.Semaphore] = None
+    ) -> AsyncGenerator[Tuple[int, bytes], None, None]:
+        """
+        Private method. Download figures geometries with given IDs from storage asynchronously.
+
+        :param ids: List of figure IDs in Supervisely.
+        :type ids: List[int]
+        :return: Async generator with pairs of figure ID and figure geometry.
+        :rtype: AsyncGenerator[Tuple[int, bytes], None, None]
+        """
+        if semaphore is None:
+            semaphore = self._api.get_default_semaphore()
+
+        for batch_ids in batched(ids):
+            async with semaphore:
+                response = await self._api.post_async(
+                    "figures.bulk.download.geometry", {ApiField.IDS: batch_ids}
+                )
+            decoder = MultipartDecoder.from_response(response)
+            for part in decoder.parts:
+                content_utf8 = part.headers[b"Content-Disposition"].decode("utf-8")
+                # Find name="1245" preceded by a whitespace, semicolon or beginning of line.
+                # The regex has 2 capture group: one for the prefix and one for the actual name value.
+                figure_id = int(re.findall(r'(^|[\s;])name="(\d*)"', content_utf8)[0][1])
+                yield figure_id, part.content
+
+    async def download_geometries_batch_async(
+        self,
+        ids: List[int],
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        semaphore: Optional[asyncio.Semaphore] = None,
+    ) -> List[dict]:
+        """
+        Download figure geometries with given IDs from storage asynchronously.
+
+        :param ids: List of figure IDs in Supervisely.
+        :type ids: List[int]
+        :param progress_cb: Progress bar to show the download progress. Shows the number of bytes downloaded.
+        :type progress_cb: Union[tqdm, Callable], optional
+        :param semaphore: Semaphore to limit the number of concurrent downloads.
+        :type semaphore: Optional[asyncio.Semaphore], optional
+        :return: List of figure geometries in Supervisely JSON format.
+        :rtype: List[dict]
+
+        :Usage example:
+
+            .. code-block:: python
+
+                import asyncio
+                import supervisely as sly
+
+                os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+                os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+                api = sly.Api.from_env()
+
+                figure_ids = [642155547, 642155548, 642155549]
+                loop = sly.utils.get_or_create_event_loop()
+                geometries = loop.run_until_complete(
+                    api.figure.download_geometries_batch_async(
+                        figure_ids,
+                        progress_cb=tqdm(total=len(figure_ids), desc="Downloading geometries"),
+                        semaphore=asyncio.Semaphore(15),
+                    )
+                )
+        """
+        geometries = {}
+        async for idx, part in self._download_geometries_generator_async(ids, semaphore):
+            if progress_cb is not None:
+                progress_cb(len(part))
+            geometry_json = json.loads(part)
+            geometries[idx] = geometry_json
+
+        if len(geometries) != len(ids):
+            raise RuntimeError("Not all geometries were downloaded")
+        ordered_results = [geometries[i] for i in ids]
+        return ordered_results

{supervisely-6.73.247 → supervisely-6.73.249}/supervisely/api/image_api.py

@@ -13,6 +13,7 @@ from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from functools import partial
+from math import ceil
 from pathlib import Path
 from time import sleep
 from typing import (
@@ -3705,11 +3706,12 @@ class ImageApi(RemoveableBulkModuleApi):
         range_start: Optional[int] = None,
         range_end: Optional[int] = None,
         headers: Optional[dict] = None,
+        chunk_size: Optional[int] = None,
     ) -> AsyncGenerator:
         """
         Download Image with given ID asynchronously.
         If is_stream is True, returns stream of bytes, otherwise returns response object.
-        For streaming, returns tuple of chunk and hash.
+        For streaming, returns tuple of chunk and hash. Chunk size is 8 MB by default.
 
         :param id: Image ID in Supervisely.
         :type id: int
@@ -3721,11 +3723,16 @@ class ImageApi(RemoveableBulkModuleApi):
         :type range_end: int, optional
         :param headers: Headers for request.
         :type headers: dict, optional
+        :param chunk_size: Size of chunk for streaming. Default is 8 MB.
+        :type chunk_size: int, optional
         :return: Stream of bytes or response object.
         :rtype: AsyncGenerator
         """
         api_method_name = "images.download"
 
+        if chunk_size is None:
+            chunk_size = 8 * 1024 * 1024
+
         json_body = {ApiField.ID: id}
 
         if is_stream:
@@ -3736,6 +3743,7 @@ class ImageApi(RemoveableBulkModuleApi):
                 headers=headers,
                 range_start=range_start,
                 range_end=range_end,
+                chunk_size=chunk_size,
             ):
                 yield chunk, hhash
         else:
@@ -4157,3 +4165,180 @@ class ImageApi(RemoveableBulkModuleApi):
             tasks.append(task)
         results = await asyncio.gather(*tasks)
         return results
+
+    async def download_bytes_generator_async(
+        self,
+        dataset_id: int,
+        img_ids: List[int],
+        semaphore: Optional[asyncio.Semaphore] = None,
+        headers: Optional[dict] = None,
+        check_hash: bool = False,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "number",
+    ) -> AsyncGenerator[Tuple[int, bytes]]:
+        """
+        Downloads Image bytes with given ID in batch asynchronously.
+        Yields tuple of Image ID and bytes of downloaded image.
+        Uses bulk download API method.
+
+        :param dataset_id: Dataset ID in Supervisely.
+        :type dataset_id: int
+        :param img_ids: List of Image IDs in Supervisely.
+        :type img_ids: :class:`List[int]`
+        :param semaphore: Semaphore for limiting the number of simultaneous downloads.
+        :type semaphore: :class:`asyncio.Semaphore`, optional
+        :param headers: Headers for request.
+        :type headers: dict, optional
+        :param check_hash: If True, checks hash of downloaded bytes. Default is False.
+        :type check_hash: bool, optional
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: Optional[Union[tqdm, Callable]]
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "number".
+        :type progress_cb_type: Literal["number", "size"], optional
+        :return: Tuple of Image ID and bytes of downloaded image.
+        :rtype: :class:`Tuple[int, bytes]`
+
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+            import asyncio
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            dataset_id = 123456
+            img_ids = [770918, 770919, 770920, 770921, ... , 770992]
+            tasks = []
+            for batch in batched(img_ids, 50):
+                task = api.image.download_bytes_batch_async(dataset_id, batch)
+                tasks.append(task)
+            results = await asyncio.gather(*tasks)
+        """
+        api_method_name = "images.bulk.download"
+        json_body = {
+            ApiField.DATASET_ID: dataset_id,
+            ApiField.IMAGE_IDS: img_ids,
+        }
+
+        if semaphore is None:
+            semaphore = self._api.get_default_semaphore()
+        async with semaphore:
+            response = await self._api.post_async(
+                api_method_name,
+                json=json_body,
+                headers=headers,
+            )
+        decoder = MultipartDecoder.from_response(response)
+        for part in decoder.parts:
+            content_utf8 = part.headers[b"Content-Disposition"].decode("utf-8")
+            # Find name="1245" preceded by a whitespace, semicolon or beginning of line.
+            # The regex has 2 capture group: one for the prefix and one for the actual name value.
+            img_id = int(re.findall(r'(^|[\s;])name="(\d*)"', content_utf8)[0][1])
+            if check_hash:
+                hhash = part.headers.get("x-content-checksum-sha256", None)
+                if hhash is not None:
+                    downloaded_bytes_hash = get_bytes_hash(part)
+                    if hhash != downloaded_bytes_hash:
+                        raise RuntimeError(
+                            f"Downloaded hash of image with ID:{img_id} does not match the expected hash: {downloaded_bytes_hash} != {hhash}"
+                        )
+            if progress_cb is not None and progress_cb_type == "number":
+                progress_cb(1)
+            elif progress_cb is not None and progress_cb_type == "size":
+                progress_cb(len(part.content))
+
+            yield img_id, part.content
+
+    async def get_list_generator_async(
+        self,
+        dataset_id: int = None,
+        filters: Optional[List[Dict[str, str]]] = None,
+        sort: Optional[str] = "id",
+        sort_order: Optional[str] = "asc",
+        force_metadata_for_links: Optional[bool] = True,
+        only_labelled: Optional[bool] = False,
+        fields: Optional[List[str]] = None,
+        per_page: Optional[int] = 500,
+        semaphore: Optional[List[asyncio.Semaphore]] = None,
+        **kwargs,
+    ) -> AsyncGenerator[List[ImageInfo]]:
+        """
+        Yields list of images in dataset asynchronously page by page.
+
+        :param dataset_id: Dataset ID in Supervisely.
+        :type dataset_id: int
+        :param filters: Filters for images.
+        :type filters: List[Dict[str, str]], optional
+        :param sort: Sort images by field.
+        :type sort: str, optional
+        :param sort_order: Sort order for images.
+        :type sort_order: str, optional
+        :param force_metadata_for_links: If True, forces metadata for links.
+        :type force_metadata_for_links: bool, optional
+        :param only_labelled: If True, returns only labelled images.
+        :type only_labelled: bool, optional
+        :param fields: List of fields to return.
+        :type fields: List[str], optional
+        :param per_page: Number of images to return per page.
+        :type per_page: int, optional
+        :param semaphore: Semaphore for limiting the number of simultaneous requests.
+        :type semaphore: :class:`asyncio.Semaphore`, optional
+        :param kwargs: Additional arguments.
+        :return: List of images in dataset.
+        :rtype: AsyncGenerator[List[ImageInfo]]
+
+        :Usage example:
+
+            .. code-block:: python
+
+                    import supervisely as sly
+                    import asyncio
+
+                    os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+                    os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+                    api = sly.Api.from_env()
+
+                    loop = sly.utils.get_or_create_event_loop()
+                    images = loop.run_until_complete(api.image.get_list_async(123456, per_page=600))
+        """
+
+        method = "images.list"
+        dataset_info = kwargs.get("dataset_info", None)
+
+        if dataset_info is None:
+            dataset_info = self._api.dataset.get_info_by_id(dataset_id, raise_error=True)
+
+        total_pages = ceil(dataset_info.items_count / per_page)
+
+        data = {
+            ApiField.DATASET_ID: dataset_info.id,
+            ApiField.PROJECT_ID: dataset_info.project_id,
+            ApiField.SORT: sort,
+            ApiField.SORT_ORDER: sort_order,
+            ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
+            ApiField.FILTER: filters or [],
+            ApiField.PER_PAGE: per_page,
+        }
+        if fields is not None:
+            data[ApiField.FIELDS] = fields
+        if only_labelled:
+            data[ApiField.FILTERS] = [
+                {
+                    "type": "objects_class",
+                    "data": {
+                        "from": 1,
+                        "to": 9999,
+                        "include": True,
+                        "classId": None,
+                    },
+                }
+            ]
+
+        if semaphore is None:
+            semaphore = self._api.get_default_semaphore()
+
+        async for page in self.get_list_page_generator_async(method, data, total_pages, semaphore):
+            yield page

{supervisely-6.73.247 → supervisely-6.73.249}/supervisely/api/module_api.py

@@ -1,7 +1,9 @@
 # coding: utf-8
+import asyncio
 from collections import namedtuple
 from copy import deepcopy
-from typing import TYPE_CHECKING, List
+from math import ceil
+from typing import TYPE_CHECKING, Any, AsyncGenerator, List, NamedTuple, Optional, Tuple
 
 import requests
 
@@ -867,6 +869,98 @@ class ModuleApiBase(_JsonConvertibleModule):
         response = self._get_response_by_id(id, method, id_field=ApiField.ID, fields=fields)
         return self._convert_json_info(response.json()) if (response is not None) else None
 
+    async def get_list_idx_page_async(
+        self,
+        method: str,
+        data: dict,
+    ) -> Tuple[int, List[NamedTuple]]:
+        """
+        Get the list of items for a given page number.
+        Page number is specified in the data dictionary.
+
+        :param method: Method to call for listing items.
+        :type method: str
+        :param data: Data to pass to the API method.
+        :type data: dict
+        :return: List of items.
+        :rtype: Tuple[int, List[NamedTuple]]
+        """
+
+        response = await self._api.post_async(method, data)
+        response_json = response.json()
+        entities = response_json.get("entities", [])
+        # To avoid empty pages when a filter is applied to the data and the `pagesCount` is less than the number calculated based on the items and `per_page` size.
+        # Process `pagesCount` in the main function according to the actual number of pages returned.
+        pages_count = response_json.get("pagesCount", None)
+        if pages_count is None:
+            raise ValueError("Can not determine the number of pages to retrieve.")
+        return pages_count, [self._convert_json_info(item) for item in entities]
+
+    async def get_list_page_generator_async(
+        self,
+        method: str,
+        data: dict,
+        pages_count: Optional[int] = None,
+        semaphore: Optional[List[asyncio.Semaphore]] = None,
+    ) -> AsyncGenerator[List[Any], None]:
+        """
+        Yields list of images in dataset asynchronously page by page.
+
+        :param method: Method to call for listing items.
+        :type method: str
+        :param data: Data to pass to the API method.
+        :type data: dict
+        :param pages_count: Preferred number of pages to retrieve if used with a `per_page` limit.
+                            Will be automatically adjusted if the `pagesCount` differs from the requested number.
+        :type pages_count: int, optional
+        :param semaphore: Semaphore for limiting the number of simultaneous requests.
+        :type semaphore: :class:`asyncio.Semaphore`, optional
+        :param kwargs: Additional arguments.
+        :return: List of images in dataset.
+        :rtype: AsyncGenerator[List[ImageInfo]]
+
+        :Usage example:
+
+            .. code-block:: python
+
+                    import supervisely as sly
+                    import asyncio
+
+                    os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+                    os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+                    api = sly.Api.from_env()
+
+                    method = 'images.list'
+                    data = {
+                        'datasetId': 123456
+                    }
+
+                    loop = sly.utils.get_or_create_event_loop()
+                    images = loop.run_until_complete(api.image.get_list_generator_async(method, data))
+        """
+
+        if semaphore is None:
+            semaphore = self._api.get_default_semaphore()
+
+        async def sem_task(task):
+            async with semaphore:
+                return await task
+
+        if pages_count is None:
+            pages_count = 999999  # to avoid range lesser than total pages count
+        for page_num in range(1, pages_count + 1):
+            if page_num <= pages_count:
+                data[ApiField.PAGE] = page_num
+                total_pages, items = await sem_task(self.get_list_idx_page_async(method, data))
+
+                # To correct `total_pages` count in case filter is applied
+                if page_num == 1:
+                    pages_count = total_pages
+
+                yield items
+            else:
+                break
+
 
 class ModuleApi(ModuleApiBase):
     """Base class for entities that have a parent object in the system."""