supervisely 6.73.226__py3-none-any.whl → 6.73.228__py3-none-any.whl

supervisely/__init__.py

@@ -55,7 +55,7 @@ from supervisely.task.progress import (
 
 import supervisely.project as project
 from supervisely.project import read_project, get_project_class
-from supervisely.project.download import download
+from supervisely.project.download import download, download_async
 from supervisely.project.upload import upload
 from supervisely.project.project import (
     Project,
@@ -309,4 +309,4 @@ except Exception as e:
 # If new changes in Supervisely Python SDK require upgrade of the Supervisely instance
 # set a new value for the environment variable MINIMUM_INSTANCE_VERSION_FOR_SDK, otherwise
 # users can face compatibility issues, if the instance version is lower than the SDK version.
-os.environ["MINIMUM_INSTANCE_VERSION_FOR_SDK"] = "6.11.16"
+os.environ["MINIMUM_INSTANCE_VERSION_FOR_SDK"] = "6.12.5"

supervisely/_utils.py

@@ -1,5 +1,6 @@
 # coding: utf-8
 
+import asyncio
 import base64
 import copy
 import hashlib
@@ -13,7 +14,7 @@ import urllib
 from datetime import datetime
 from functools import wraps
 from tempfile import gettempdir
-from typing import List, Literal, Optional
+from typing import Any, Dict, List, Literal, Optional, Tuple
 
 import numpy as np
 from requests.utils import DEFAULT_CA_BUNDLE_PATH
@@ -315,6 +316,15 @@ def get_readable_datetime(value: str) -> str:
     return dt.strftime("%Y-%m-%d %H:%M:%S")
 
 
+def get_unix_timestamp() -> int:
+    """Return the current Unix timestamp.
+
+    :return: Current Unix timestamp.
+    :rtype: int
+    """
+    return int(time.time())
+
+
 def get_certificates_list(path: str = DEFAULT_CA_BUNDLE_PATH) -> List[str]:
     with open(path, "r", encoding="ascii") as f:
         content = f.read().strip()
@@ -382,3 +392,70 @@ def add_callback(func, callback):
         return res
 
     return wrapper
+
+
+def compare_dicts(
+    template: Dict[Any, Any], data: Dict[Any, Any], strict: bool = True
+) -> Tuple[List[str], List[str]]:
+    """Compare two dictionaries recursively (by keys only) and return lists of missing and extra fields.
+    If strict is True, the keys of the template and data dictionaries must match exactly.
+    Otherwise, the data dictionary may contain additional keys that are not in the template dictionary.
+
+    :param template: The template dictionary.
+    :type template: Dict[Any, Any]
+    :param data: The data dictionary.
+    :type data: Dict[Any, Any]
+    :param strict: If True, the keys of the template and data dictionaries must match exactly.
+    :type strict: bool, optional
+    :return: A tuple containing a list of missing fields and a list of extra fields.
+    :rtype: Tuple[List[str], List[str]]
+    """
+    missing_fields = []
+    extra_fields = []
+
+    if not isinstance(template, dict) or not isinstance(data, dict):
+        return missing_fields, extra_fields
+
+    if strict:
+        template_keys = set(template.keys())
+        data_keys = set(data.keys())
+
+        missing_fields = list(template_keys - data_keys)
+        extra_fields = list(data_keys - template_keys)
+
+        for key in template_keys & data_keys:
+            sub_missing, sub_extra = compare_dicts(template[key], data[key], strict)
+            missing_fields.extend([f"{key}.{m}" for m in sub_missing])
+            extra_fields.extend([f"{key}.{e}" for e in sub_extra])
+    else:
+        for key in template:
+            if key not in data:
+                missing_fields.append(key)
+            else:
+                sub_missing, sub_extra = compare_dicts(template[key], data[key], strict)
+                missing_fields.extend([f"{key}.{m}" for m in sub_missing])
+                extra_fields.extend([f"{key}.{e}" for e in sub_extra])
+
+    return missing_fields, extra_fields
+
+
+def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
+    """
+    Get the current event loop or create a new one if it doesn't exist.
+    Works for different Python versions and contexts.
+
+    :return: Event loop
+    :rtype: asyncio.AbstractEventLoop
+    """
+    try:
+        # Preferred method for asynchronous context (Python 3.7+)
+        return asyncio.get_running_loop()
+    except RuntimeError:
+        # If the loop is not running, get the current one or create a new one (Python 3.8 and 3.9)
+        try:
+            return asyncio.get_event_loop()
+        except RuntimeError:
+            # For Python 3.10+ or if the call occurs outside of an active loop context
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+            return loop

supervisely/api/annotation_api.py

@@ -4,10 +4,11 @@
 # docs
 from __future__ import annotations
 
+import asyncio
 import json
 from collections import defaultdict
 from copy import deepcopy
-from typing import Any, Callable, Dict, List, NamedTuple, Optional, Union
+from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Union
 
 from tqdm import tqdm
 
@@ -1307,36 +1308,36 @@ class AnnotationApi(ModuleApi):
         Priority increases with the number: a higher number indicates a higher priority.
         The higher priority means that the label will be displayed on top of the others.
         The lower priority means that the label will be displayed below the others.
-        
+
         :param label_id: ID of the label to update
         :type label_id: int
         :param priority: New priority of the label
         :type priority: int
-        
-        :Usage example: 
-        
+
+        :Usage example:
+
             .. code-block:: python
-            
+
             import os
             from dotenv import load_dotenv
-            
+
             import supervisely as sly
-            
+
             # Load secrets and create API object from .env file (recommended)
             # Learn more here: https://developer.supervisely.com/getting-started/basics-of-authentication
             load_dotenv(os.path.expanduser("~/supervisely.env"))
-            
+
             api = sly.Api.from_env()
-            
+
             label_ids = [123, 456, 789]
             priorities = [1, 2, 3]
-            
+
             for label_id, priority in zip(label_ids, priorities):
                 api.annotation.update_label_priority(label_id, priority)
-                
+
             # The label with ID 789 will be displayed on top of the others.
             # The label with ID 123 will be displayed below the others.
-        
+
         """
         self._api.post(
             "figures.priority.update",
@@ -1344,4 +1345,173 @@ class AnnotationApi(ModuleApi):
                 ApiField.ID: label_id,
                 ApiField.PRIORITY: priority,
             },
-        )
+        )
+
+    async def download_async(
+        self,
+        image_id: int,
+        semaphore: Optional[asyncio.Semaphore] = None,
+        with_custom_data: Optional[bool] = False,
+        force_metadata_for_links: Optional[bool] = True,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "number",
+    ) -> AnnotationInfo:
+        """
+        Download AnnotationInfo by image ID from API.
+
+        :param image_id: Image ID in Supervisely.
+        :type image_id: int
+        :param semaphore: Semaphore for limiting the number of simultaneous downloads.
+        :type semaphore: asyncio.Semaphore, optional
+        :param with_custom_data: Include custom data in the response.
+        :type with_custom_data: bool, optional
+        :param force_metadata_for_links: Force metadata for links.
+        :type force_metadata_for_links: bool, optional
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: tqdm or callable, optional
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "number".
+        :type progress_cb_type: str, optional
+        :return: Information about Annotation. See :class:`info_sequence<info_sequence>`
+        :rtype: :class:`AnnotationInfo`
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            image_id = 121236918
+            loop = sly.utils.get_or_create_event_loop()
+            ann_info = loop.run_until_complete(api.annotation.download_async(image_id))
+        """
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+        async with semaphore:
+            response = await self._api.post_async(
+                "annotations.info",
+                {
+                    ApiField.IMAGE_ID: image_id,
+                    ApiField.WITH_CUSTOM_DATA: with_custom_data,
+                    ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
+                    ApiField.INTEGER_COORDS: False,
+                },
+            )
+            if progress_cb is not None and progress_cb_type == "size":
+                progress_cb(len(response.content))
+
+            result = response.json()
+            # Convert annotation to pixel coordinate system
+            result[ApiField.ANNOTATION] = Annotation._to_pixel_coordinate_system_json(
+                result[ApiField.ANNOTATION]
+            )
+            # check if there are any AlphaMask geometries in the batch
+            additonal_geometries = defaultdict(int)
+            labels = result[ApiField.ANNOTATION][AnnotationJsonFields.LABELS]
+            for idx, label in enumerate(labels):
+                if label[LabelJsonFields.GEOMETRY_TYPE] == AlphaMask.geometry_name():
+                    figure_id = label[LabelJsonFields.ID]
+                    additonal_geometries[figure_id] = idx
+
+            # if so, download them separately and update the annotation
+            if len(additonal_geometries) > 0:
+                figure_ids = list(additonal_geometries.keys())
+                figures = self._api.image.figure.download_geometries_batch(
+                    figure_ids,
+                    (
+                        progress_cb
+                        if progress_cb is not None and progress_cb_type == "size"
+                        else None
+                    ),
+                )
+                for figure_id, geometry in zip(figure_ids, figures):
+                    label_idx = additonal_geometries[figure_id]
+                    labels[label_idx].update({BITMAP: geometry})
+            ann_info = self._convert_json_info(result)
+            if progress_cb is not None and progress_cb_type == "number":
+                progress_cb(1)
+            return ann_info
+
+    async def download_batch_async(
+        self,
+        dataset_id: int,
+        image_ids: List[int],
+        semaphore: Optional[asyncio.Semaphore] = None,
+        with_custom_data: Optional[bool] = False,
+        force_metadata_for_links: Optional[bool] = True,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "number",
+    ) -> List[AnnotationInfo]:
+        """
+        Get list of AnnotationInfos for given dataset ID from API.
+
+        :param dataset_id: Dataset ID in Supervisely.
+        :type dataset_id: int
+        :param image_ids: List of integers.
+        :type image_ids: List[int]
+        :param semaphore: Semaphore for limiting the number of simultaneous downloads.
+        :type semaphore: asyncio.Semaphore, optional
+        :param with_custom_data: Include custom data in the response.
+        :type with_custom_data: bool, optional
+        :param force_metadata_for_links: Force metadata for links.
+        :type force_metadata_for_links: bool, optional
+        :param progress_cb: Function for tracking download progress. Total should be equal to len(image_ids) or None.
+        :type progress_cb: tqdm or callable, optional
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "number".
+        :type progress_cb_type: str, optional
+        :return: Information about Annotations. See :class:`info_sequence<info_sequence>`
+        :rtype: :class:`List[AnnotationInfo]`
+
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            dataset_id = 254737
+            image_ids = [121236918, 121236919]
+            pbar = tqdm(desc="Download annotations", total=len(image_ids))
+
+            loop = sly.utils.get_or_create_event_loop()
+            ann_infos = loop.run_until_complete(
+                                api.annotation.download_batch_async(dataset_id, image_ids, progress_cb=pbar)
+                            )
+        """
+
+        # use context to avoid redundant API calls
+        context = self._api.optimization_context
+        context_dataset_id = context.get("dataset_id")
+        project_meta = context.get("project_meta")
+        project_id = context.get("project_id")
+        if dataset_id != context_dataset_id:
+            context["dataset_id"] = dataset_id
+            project_id, project_meta = None, None
+
+        if not isinstance(project_meta, ProjectMeta):
+            if project_id is None:
+                project_id = self._api.dataset.get_info_by_id(dataset_id).project_id
+                context["project_id"] = project_id
+            project_meta = ProjectMeta.from_json(self._api.project.get_meta(project_id))
+            context["project_meta"] = project_meta
+
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+        tasks = []
+        for image in image_ids:
+            task = self.download_async(
+                image_id=image,
+                semaphore=semaphore,
+                with_custom_data=with_custom_data,
+                force_metadata_for_links=force_metadata_for_links,
+                progress_cb=progress_cb,
+                progress_cb_type=progress_cb_type,
+            )
+            tasks.append(task)
+        ann_infos = await asyncio.gather(*tasks)
+        return ann_infos

supervisely/api/api.py

@@ -1248,7 +1248,7 @@ class Api:
                         raise ValueError(
                             f"Streamed size does not match the expected: {total_streamed} != {expected_size}"
                         )
-                    logger.debug(f"Streamed size: {total_streamed}, expected size: {expected_size}")
+                    logger.trace(f"Streamed size: {total_streamed}, expected size: {expected_size}")
                     return
             except httpx.RequestError as e:
                 retry_range_start = total_streamed + (range_start or 0)
@@ -1478,7 +1478,7 @@ class Api:
                         raise ValueError(
                             f"Streamed size does not match the expected: {total_streamed} != {expected_size}"
                         )
-                    logger.debug(f"Streamed size: {total_streamed}, expected size: {expected_size}")
+                    logger.trace(f"Streamed size: {total_streamed}, expected size: {expected_size}")
                     return
             except httpx.RequestError as e:
                 retry_range_start = total_streamed + (range_start or 0)

supervisely/api/dataset_api.py

@@ -4,7 +4,17 @@
 # docs
 from __future__ import annotations
 
-from typing import Dict, Generator, List, Literal, NamedTuple, Optional, Tuple, Union
+from typing import (
+    Any,
+    Dict,
+    Generator,
+    List,
+    Literal,
+    NamedTuple,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from supervisely._utils import abs_url, compress_image_url, is_development
 from supervisely.api.module_api import (
@@ -61,6 +71,7 @@ class DatasetInfo(NamedTuple):
     team_id: int
     workspace_id: int
     parent_id: Union[int, None]
+    custom_data: dict
 
     @property
     def image_preview_url(self):
@@ -135,6 +146,7 @@ class DatasetApi(UpdateableModule, RemoveableModuleApi):
             ApiField.TEAM_ID,
             ApiField.WORKSPACE_ID,
             ApiField.PARENT_ID,
+            ApiField.CUSTOM_DATA,
         ]
 
     @staticmethod
@@ -362,6 +374,83 @@ class DatasetApi(UpdateableModule, RemoveableModuleApi):
             )
         return dataset_info
 
+    def update(
+        self,
+        id: int,
+        name: Optional[str] = None,
+        description: Optional[str] = None,
+        custom_data: Optional[Dict[Any, Any]] = None,
+    ) -> DatasetInfo:
+        """Update Dataset information by given ID.
+
+        :param id: Dataset ID in Supervisely.
+        :type id: int
+        :param name: New Dataset name.
+        :type name: str, optional
+        :param description: New Dataset description.
+        :type description: str, optional
+        :param custom_data: New custom data.
+        :type custom_data: Dict[Any, Any], optional
+        :return: Information about Dataset. See :class:`info_sequence<info_sequence>`
+        :rtype: :class:`DatasetInfo`
+
+        :Usage example:
+
+             .. code-block:: python
+
+                import supervisely as sly
+
+                dataset_id = 384126
+
+                os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+                os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+                api = sly.Api.from_env()
+
+                new_ds = api.dataset.update(dataset_id, name='new_ds', description='new description')
+        """
+        fields = [name, description, custom_data]  # Extend later if needed.
+        if all(f is None for f in fields):
+            raise ValueError(f"At least one of the fields must be specified: {fields}")
+
+        payload = {
+            ApiField.ID: id,
+            ApiField.NAME: name,
+            ApiField.DESCRIPTION: description,
+            ApiField.CUSTOM_DATA: custom_data,
+        }
+
+        payload = {k: v for k, v in payload.items() if v is not None}
+
+        response = self._api.post(self._get_update_method(), payload)
+        return self._convert_json_info(response.json())
+
+    def update_custom_data(self, id: int, custom_data: Dict[Any, Any]) -> DatasetInfo:
+        """Update custom data for Dataset by given ID.
+        Custom data is a dictionary that can store any additional information about the Dataset.
+
+        :param id: Dataset ID in Supervisely.
+        :type id: int
+        :param custom_data: New custom data.
+        :type custom_data: Dict[Any, Any]
+        :return: Information about Dataset. See :class:`info_sequence<info_sequence>`
+        :rtype: :class:`DatasetInfo`
+
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            dataset_id = 384126
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            new_ds = api.dataset.update_custom_data(dataset_id, custom_data={'key': 'value'})
+        """
+        return self.update(id, custom_data=custom_data)
+
     def _get_update_method(self):
         """ """
         return "datasets.editInfo"

supervisely/api/entity_annotation/figure_api.py

@@ -6,10 +6,11 @@ from __future__ import annotations
 import json
 import re
 from collections import defaultdict
-from typing import Dict, Generator, List, NamedTuple, Optional, Tuple
+from typing import Callable, Dict, Generator, List, NamedTuple, Optional, Tuple, Union
 
 import numpy as np
 from requests_toolbelt import MultipartDecoder, MultipartEncoder
+from tqdm import tqdm
 
 from supervisely._utils import batched
 from supervisely.api.module_api import ApiField, ModuleApi, RemoveableBulkModuleApi
@@ -540,17 +541,25 @@ class FigureApi(RemoveableBulkModuleApi):
         """
         return self.download_geometries_batch([figure_id])
 
-    def download_geometries_batch(self, ids: List[int]) -> List[dict]:
+    def download_geometries_batch(
+        self,
+        ids: List[int],
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+    ) -> List[dict]:
         """
         Download figure geometries with given IDs from storage.
 
         :param ids: List of figure IDs in Supervisely.
         :type ids: List[int]
+        :param progress_cb: Progress bar to show the download progress. Shows the number of bytes downloaded.
+        :type progress_cb: Union[tqdm, Callable], optional
         :return: List of figure geometries in Supervisely JSON format.
         :rtype: List[dict]
         """
         geometries = {}
         for idx, part in self._download_geometries_generator(ids):
+            if progress_cb is not None:
+                progress_cb(len(part.content))
             geometry_json = json.loads(part.content)
             geometries[idx] = geometry_json