supervisely 6.73.225__py3-none-any.whl → 6.73.227__py3-none-any.whl

supervisely/api/video/video_annotation_api.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import annotations
 
+import asyncio
 import json
 from typing import Callable, Dict, List, Optional, Union
 
@@ -247,3 +248,47 @@ class VideoAnnotationAPI(EntityAnnotationAPI):
                 self.append(dst_id, ann)
                 if progress_cb is not None:
                     progress_cb(1)
+
+    async def download_async(
+        self,
+        video_id: int,
+        video_info=None,
+        semaphore: Optional[asyncio.Semaphore] = None,
+    ) -> Dict:
+        """
+        Download information about VideoAnnotation by video ID from API asynchronously.
+
+        :param video_id: Video ID in Supervisely.
+        :type video_id: int
+        :param video_info: VideoInfo object. Use it to avoid additional request to the server.
+        :type video_info: VideoInfo, optional
+        :param semaphore: Semaphore to limit the number of parallel downloads.
+        :type semaphore: asyncio.Semaphore, optional
+        :return: Information about VideoAnnotation in json format
+        :rtype: :class:`dict`
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            video_id = 198702499
+            loop = sly.utils.get_or_create_event_loop()
+            ann_info = loop.run_until_complete(api.video.annotation.download_async(video_id))
+        """
+        if video_info is None:
+            video_info = self._api.video.get_info_by_id(video_id)
+
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+
+        async with semaphore:
+            response = await self._api.post_async(
+                self._method_download_bulk,
+                {ApiField.DATASET_ID: video_info.dataset_id, self._entity_ids_str: [video_info.id]},
+            )
+            return response.json()

supervisely/api/video/video_api.py

@@ -2466,8 +2466,7 @@ class VideoApi(RemoveableBulkModuleApi):
             save_path = os.path.join("/path/to/save/", video_info.name)
 
             semaphore = asyncio.Semaphore(100)
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                     api.video.download_path_async(video_info.id, save_path, semaphore)
                 )
@@ -2555,8 +2554,7 @@ class VideoApi(RemoveableBulkModuleApi):
 
             ids = [770914, 770915]
             paths = ["/path/to/save/video1.mp4", "/path/to/save/video2.mp4"]
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(api.video.download_paths_async(ids, paths))
         """
         if len(ids) == 0:

supervisely/api/volume/volume_api.py

@@ -1343,8 +1343,7 @@ class VolumeApi(RemoveableBulkModuleApi):
             save_path = os.path.join("/path/to/save/", volume_info.name)
 
             semaphore = asyncio.Semaphore(100)
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                         api.volume.download_path_async(volume_info.id, save_path, semaphore)
                 )
@@ -1433,8 +1432,7 @@ class VolumeApi(RemoveableBulkModuleApi):
 
             ids = [770914, 770915]
             paths = ["/path/to/save/volume1.nrrd", "/path/to/save/volume2.nrrd"]
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(api.volume.download_paths_async(ids, paths))
         """
         if len(ids) == 0:

supervisely/convert/base_converter.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Tuple, Union
 
 from tqdm import tqdm
 
-from supervisely._utils import batched, is_production
+from supervisely._utils import batched, get_or_create_event_loop, is_production
 from supervisely.annotation.annotation import Annotation
 from supervisely.annotation.tag_meta import TagValueType
 from supervisely.api.api import Api
@@ -468,7 +468,7 @@ class BaseConverter:
                     for remote_path in files.values()
                 )
 
-            loop = asyncio.get_event_loop()
+            loop = get_or_create_event_loop()
             _, progress_cb = self.get_progress(
                 len(files) if not is_archive_type else file_size,
                 f"Downloading {files_type} from remote storage",
@@ -479,15 +479,19 @@ class BaseConverter:
                 silent_remove(local_path)
 
             logger.info(f"Downloading {files_type} from remote storage...")
-            loop.run_until_complete(
-                self._api.storage.download_bulk_async(
-                    team_id=self._team_id,
-                    remote_paths=list(files.values()),
-                    local_save_paths=list(files.keys()),
-                    progress_cb=progress_cb,
-                    progress_cb_type="number" if not is_archive_type else "size",
-                )
+            download_coro = self._api.storage.download_bulk_async(
+                team_id=self._team_id,
+                remote_paths=list(files.values()),
+                local_save_paths=list(files.keys()),
+                progress_cb=progress_cb,
+                progress_cb_type="number" if not is_archive_type else "size",
             )
+
+            if loop.is_running():
+                future = asyncio.run_coroutine_threadsafe(download_coro, loop=loop)
+                future.result()
+            else:
+                loop.run_until_complete(download_coro)
             logger.info("Possible annotations downloaded successfully.")
 
             if is_archive_type:

supervisely/io/fs.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 
 # docs
-import asyncio
 import errno
 import mimetypes
 import os
@@ -16,7 +15,7 @@ import requests
 from requests.structures import CaseInsensitiveDict
 from tqdm import tqdm
 
-from supervisely._utils import get_bytes_hash, get_string_hash
+from supervisely._utils import get_bytes_hash, get_or_create_event_loop, get_string_hash
 from supervisely.io.fs_cache import FileCache
 from supervisely.sly_logger import logger
 from supervisely.task.progress import Progress
@@ -1375,8 +1374,15 @@ async def copy_file_async(
 
      .. code-block:: python
 
-        from supervisely.io.fs import async_copy_file
-        await async_copy_file('/home/admin/work/projects/example/1.png', '/home/admin/work/tests/2.png')
+        import supervisely as sly
+
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.copy_file_async('/home/admin/work/projects/example/1.png', '/home/admin/work/tests/2.png')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
     """
     ensure_base_path(dst)
     async with aiofiles.open(dst, "wb") as out_f:
@@ -1404,8 +1410,15 @@ async def get_file_hash_async(path: str) -> str:
 
      .. code-block:: python
 
-        from supervisely.io.fs import get_file_hash_async
-        hash = await get_file_hash_async('/home/admin/work/projects/examples/1.jpeg') # rKLYA/p/P64dzidaQ/G7itxIz3ZCVnyUhEE9fSMGxU4=
+        import supervisely as sly
+
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.get_file_hash_async('/home/admin/work/projects/examples/1.jpeg')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            hash = future.result()
+        else:
+            hash = loop.run_until_complete(coro)
     """
     async with aiofiles.open(path, "rb") as file:
         file_bytes = await file.read()
@@ -1442,7 +1455,13 @@ async def unpack_archive_async(
         archive_path = '/home/admin/work/examples.tar'
         target_dir = '/home/admin/work/projects'
 
-        await sly.fs.unpack_archive(archive_path, target_dir)
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.unpack_archive_async(archive_path, target_dir)
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
     """
     if is_split:
         chunk = chunk_size_mb * 1024 * 1024
@@ -1467,9 +1486,37 @@ async def unpack_archive_async(
                         await output_file.write(data)
         archive_path = combined
 
-    loop = asyncio.get_running_loop()
+    loop = get_or_create_event_loop()
     await loop.run_in_executor(None, shutil.unpack_archive, archive_path, target_dir)
     if is_split:
         silent_remove(archive_path)
     if remove_junk:
         remove_junk_from_dir(target_dir)
+
+
+async def touch_async(path: str) -> None:
+    """
+    Sets access and modification times for a file asynchronously.
+
+    :param path: Target file path.
+    :type path: str
+    :returns: None
+    :rtype: :class:`NoneType`
+    :Usage example:
+
+     .. code-block:: python
+
+        import supervisely as sly
+
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.touch_async('/home/admin/work/projects/examples/1.jpeg')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
+    """
+    ensure_base_path(path)
+    async with aiofiles.open(path, "a"):
+        loop = get_or_create_event_loop()
+        await loop.run_in_executor(None, os.utime, path, None)

supervisely/io/json.py

@@ -3,6 +3,7 @@ import json
 import os
 from typing import Dict, Optional
 
+import aiofiles
 import jsonschema
 
 
@@ -230,3 +231,34 @@ def validate_json(data: Dict, schema: Dict, raise_error: bool = False) -> bool:
         if raise_error:
             raise ValueError("JSON data is invalid. See error message for more details.") from err
         return False
+
+
+async def dump_json_file_async(data: Dict, filename: str, indent: Optional[int] = 4) -> None:
+    """
+    Write given data in json format in file with given name asynchronously.
+
+    :param data: Data in json format as a dict.
+    :type data: dict
+    :param filename: Target file path to write data.
+    :type filename: str
+    :param indent: Json array elements and object members will be pretty-printed with that indent level.
+    :type indent: int, optional
+    :returns: None
+    :rtype: :class:`NoneType`
+    :Usage example:
+
+     .. code-block:: python
+
+        import supervisely as sly
+
+        data = {1: 'example'}
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.json.dump_json_file_async(data, '/home/admin/work/projects/examples/1.json')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
+    """
+    async with aiofiles.open(filename, "w") as fout:
+        await fout.write(json.dumps(data, indent=indent))

supervisely/nn/inference/inference.py

@@ -574,10 +574,15 @@ class Inference:
         for prediction in predictions:
             if (
                 not classes_whitelist in (None, "all")
+                and hasattr(prediction, "class_name")
                 and prediction.class_name not in classes_whitelist
             ):
                 continue
-            label = self._create_label(prediction)
+            if "classes_whitelist" in inspect.signature(self._create_label).parameters:
+                # pylint: disable=unexpected-keyword-arg
+                label = self._create_label(prediction, classes_whitelist)  # pylint: disable=too-many-function-args
+            else:
+                label = self._create_label(prediction)
             if label is None:
                 # for example empty mask
                 continue
@@ -2543,17 +2548,52 @@ def clean_up_cuda():
         logger.debug("Error in clean_up_cuda.", exc_info=True)
 
 
+def _fix_classes_names(meta: ProjectMeta, ann: Annotation):
+    def _replace_strip(s, chars: str, replacement: str = "_") -> str:
+        replace_pattern = f"^[{re.escape(chars)}]+|[{re.escape(chars)}]+$"
+        return re.sub(replace_pattern, replacement, s)
+
+    replaced_classes_in_meta = []
+    for obj_class in meta.obj_classes:
+        obj_class_name = _replace_strip(obj_class.name, " ", "")
+        if obj_class_name != obj_class.name:
+            new_obj_class = obj_class.clone(name=obj_class_name)
+            meta = meta.delete_obj_class(obj_class.name)
+            meta = meta.add_obj_class(new_obj_class)
+            replaced_classes_in_meta.append((obj_class.name, obj_class_name))
+    replaced_classes_in_ann = set()
+    new_labels = []
+    for label in ann.labels:
+        obj_class = label.obj_class
+        obj_class_name = _replace_strip(obj_class.name, " ", "")
+        if obj_class_name != obj_class.name:
+            new_obj_class = obj_class.clone(name=obj_class_name)
+            label = label.clone(obj_class=new_obj_class)
+            replaced_classes_in_ann.add((obj_class.name, obj_class_name))
+        new_labels.append(label)
+    ann = ann.clone(labels=new_labels)
+    return meta, ann, replaced_classes_in_meta, list(replaced_classes_in_ann)
+
+
 def update_meta_and_ann(meta: ProjectMeta, ann: Annotation):
     """Update project meta and annotation to match each other
     If obj class or tag meta from annotation conflicts with project meta
     add suffix to obj class or tag meta.
     Return tuple of updated project meta, annotation and boolean flag if meta was changed."""
-    obj_classes_suffixes = {"_nn"}
-    tag_meta_suffixes = {"_nn"}
+    obj_classes_suffixes = ["_nn"]
+    tag_meta_suffixes = ["_nn"]
     ann_obj_classes = {}
     ann_tag_metas = {}
     meta_changed = False
 
+    meta, ann, replaced_classes_in_meta, replaced_classes_in_ann = _fix_classes_names(meta, ann)
+    if replaced_classes_in_meta:
+        meta_changed = True
+        logger.warning(
+            "Some classes names were fixed in project meta",
+            extra={"replaced_classes": {old: new for old, new in replaced_classes_in_meta}},
+        )
+
     # get all obj classes and tag metas from annotation
     for label in ann.labels:
         ann_obj_classes[label.obj_class.name] = label.obj_class
@@ -2563,7 +2603,8 @@ def update_meta_and_ann(meta: ProjectMeta, ann: Annotation):
         ann_tag_metas[tag.meta.name] = tag.meta
 
     # check if obj classes are in project meta
-    # if not, add them with suffix
+    # if not, add them.
+    # if shape is different, add them with suffix
     changed_obj_classes = {}
     for ann_obj_class in ann_obj_classes.values():
         if meta.get_obj_class(ann_obj_class.name) is None:

supervisely/nn/inference/semantic_segmentation/semantic_segmentation.py

@@ -3,9 +3,11 @@ from typing import Any, Dict, List
 import numpy as np
 
 from supervisely.annotation.label import Label
+from supervisely.annotation.obj_class import ObjClass
 from supervisely.geometry.bitmap import Bitmap
 from supervisely.nn.inference.inference import Inference
 from supervisely.nn.prediction_dto import PredictionSegmentation
+from supervisely.project.project_meta import ProjectMeta
 from supervisely.sly_logger import logger
 
 
@@ -24,20 +26,48 @@ class SemanticSegmentation(Inference):
     def _get_obj_class_shape(self):
         return Bitmap
 
-    def _create_label(self, dto: PredictionSegmentation):
-        image_classes = np.unique(dto.mask)
+    def _find_bg_class_index(self, class_names: List[str]):
+        possible_bg_names = ["background", "bg", "unlabeled", "neutral", "__bg__"]
+        bg_class_index = None
+        for i, name in enumerate(class_names):
+            if name in possible_bg_names:
+                bg_class_index = i
+                break
+        return bg_class_index
+
+    def _add_default_bg_class(self, meta: ProjectMeta):
+        default_bg_class_name = "__bg__"
+        obj_class = meta.get_obj_class(default_bg_class_name)
+        if obj_class is None:
+            obj_class = ObjClass(default_bg_class_name, self._get_obj_class_shape())
+            meta = meta.add_obj_class(obj_class)
+        return meta, obj_class
+
+    def _get_or_create_bg_obj_class(self, classes):
+        bg_class_index = self._find_bg_class_index(classes)
+        if bg_class_index is None:
+            self._model_meta, bg_obj_class = self._add_default_bg_class(self.model_meta)
+        else:
+            bg_class_name = classes[bg_class_index]
+            bg_obj_class = self.model_meta.get_obj_class(bg_class_name)
+        return bg_obj_class
+
+    def _create_label(self, dto: PredictionSegmentation, classes_whitelist: List[str] = None):
+        classes = self.get_classes()
+
+        image_classes_indexes = np.unique(dto.mask)
         labels = []
-        for class_idx in image_classes:
+        for class_idx in image_classes_indexes:
             class_mask = dto.mask == class_idx
-            class_name = self.get_classes()[class_idx]
-            obj_class = self.model_meta.get_obj_class(class_name)
+            class_name = classes[class_idx]
+            if classes_whitelist not in (None, "all") and class_name not in classes_whitelist:
+                obj_class = self._get_or_create_bg_obj_class(classes)
+            else:
+                obj_class = self.model_meta.get_obj_class(class_name)
             if obj_class is None:
                 raise KeyError(
                     f"Class {class_name} not found in model classes {self.get_classes()}"
                 )
-            if not class_mask.any():  # skip empty masks
-                logger.debug(f"Mask of class {class_name} is empty and will be sklipped")
-                return None
             geometry = Bitmap(class_mask, extra_validation=False)
             label = Label(geometry, obj_class)
             labels.append(label)