supervisely 6.73.226__py3-none-any.whl → 6.73.228__py3-none-any.whl

supervisely/api/project_api.py

@@ -4,7 +4,9 @@
 # docs
 from __future__ import annotations
 
+import os
 from collections import defaultdict
+from copy import deepcopy
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -25,7 +27,13 @@ if TYPE_CHECKING:
 from datetime import datetime, timedelta
 
 from supervisely import logger
-from supervisely._utils import abs_url, compress_image_url, is_development
+from supervisely._utils import (
+    abs_url,
+    compare_dicts,
+    compress_image_url,
+    get_unix_timestamp,
+    is_development,
+)
 from supervisely.annotation.annotation import TagCollection
 from supervisely.annotation.obj_class import ObjClass
 from supervisely.annotation.obj_class_collection import ObjClassCollection
@@ -36,6 +44,7 @@ from supervisely.api.module_api import (
     RemoveableModuleApi,
     UpdateableModule,
 )
+from supervisely.io.json import dump_json_file, load_json_file
 from supervisely.project.project_meta import ProjectMeta
 from supervisely.project.project_meta import ProjectMetaJsonFields as MetaJsonF
 from supervisely.project.project_settings import (
@@ -43,6 +52,9 @@ from supervisely.project.project_settings import (
     ProjectSettingsJsonFields,
 )
 from supervisely.project.project_type import (
+    _METADATA_SYSTEM_KEY,
+    _METADATA_TIMESTAMP_KEY,
+    _METADATA_VALIDATION_SCHEMA_KEY,
     _MULTISPECTRAL_TAG_NAME,
     _MULTIVIEW_TAG_NAME,
     ProjectType,
@@ -969,6 +981,278 @@ class ProjectApi(CloneableModuleApi, UpdateableModule, RemoveableModuleApi):
         )
         return response.json()
 
+    def get_custom_data(self, id: int) -> Dict[Any, Any]:
+        """Returns custom data of the Project by ID.
+        Custom data is a dictionary that can be used to store any additional information.
+
+        :param id: Project ID in Supervisely.
+        :type id: int
+        :return: Custom data of the Project
+        :rtype: :class:`dict`
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            api = sly.Api.from_env()
+
+            project_id = 123456
+
+            custom_data = api.project.get_custom_data(project_id)
+
+            print(custom_data) # Output: {'key': 'value'}
+        """
+        return self.get_info_by_id(id).custom_data
+
+    def _get_system_custom_data(self, id: int) -> Dict[Any, Any]:
+        """Returns system custom data of the Project by ID.
+        System custom data is just a part of custom data that is used to store system information
+        and obtained by the key `_METADATA_SYSTEM_KEY`.
+
+        :param id: Project ID in Supervisely.
+        :type id: int
+        :return: System custom data of the Project
+        :rtype: :class:`dict`
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            api = sly.Api.from_env()
+
+            project_id = 123456
+
+            system_custom_data = api.project._get_system_custom_data(project_id)
+
+            print(system_custom_data)
+        """
+        return self.get_info_by_id(id).custom_data.get(_METADATA_SYSTEM_KEY, {})
+
+    def get_validation_schema(self, id: int, use_caching: bool = False) -> Optional[Dict[Any, Any]]:
+        """Returns validation schema of the Project by ID.
+        Validation schema is a dictionary that can be used to validate metadata of each entity in the project
+        if corresnpoding schema is provided.
+        If using caching, the schema will be loaded from the cache if available.
+        Use cached version only in scenarios when the schema is not expected to change,
+        otherwise it may lead to checks with outdated schema.
+
+        :param id: Project ID in Supervisely.
+        :type id: int
+        :param use_caching: If True, uses cached version of the schema if available.
+            NOTE: This may lead to checks with outdated schema. Use with caution.
+            And only in scenarios when the schema is not expected to change.
+        :return: Validation schema of the Project
+        :rtype: :class:`dict`
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            api = sly.Api.from_env()
+
+            project_id = 123456
+
+            validation_schema = api.project.get_validation_schema(project_id)
+
+            print(validation_schema) # Output: {'key': 'Description of the field'}
+        """
+        SCHEMA_DIFF_THRESHOLD = 60 * 60  # 1 hour
+        json_cache_filename = os.path.join(os.getcwd(), f"{id}_validation_schema.json")
+
+        if use_caching:
+            if os.path.isfile(json_cache_filename):
+                try:
+                    schema = load_json_file(json_cache_filename)
+                    timestamp = schema.pop(_METADATA_TIMESTAMP_KEY, 0)
+
+                    if get_unix_timestamp() - timestamp < SCHEMA_DIFF_THRESHOLD:
+                        return schema
+                except RuntimeError:
+                    pass
+
+        schema = self._get_system_custom_data(id).get(_METADATA_VALIDATION_SCHEMA_KEY)
+        if schema and use_caching:
+            schema_with_timestamp = deepcopy(schema)
+            schema_with_timestamp[_METADATA_TIMESTAMP_KEY] = get_unix_timestamp()
+            dump_json_file(schema_with_timestamp, json_cache_filename)
+
+        return schema
+
+    def _edit_validation_schema(
+        self, id: int, schema: Optional[Dict[Any, Any]] = None
+    ) -> Dict[Any, Any]:
+        """Edits validation schema of the Project by ID.
+        Do not use this method directly, use `set_validation_schema` or `remove_validation_schema` instead.
+
+        :param id: Project ID in Supervisely.
+        :type id: int
+        :param schema: Validation schema to set. If None, removes validation schema.
+        :type schema: dict, optional
+        :return: Project information in dict format
+        :rtype: :class:`dict`
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            api = sly.Api.from_env()
+
+            project_id = 123456
+
+            schema = {'key': 'Description of the field'}
+
+            api.project._edit_validation_schema(project_id, schema) #Set new validation schema.
+            api.project._edit_validation_schema(project_id) #Remove validation schema.
+        """
+        custom_data = self.get_custom_data(id)
+        system_data = custom_data.setdefault(_METADATA_SYSTEM_KEY, {})
+
+        if not schema:
+            system_data.pop(_METADATA_VALIDATION_SCHEMA_KEY, None)
+        else:
+            system_data[_METADATA_VALIDATION_SCHEMA_KEY] = schema
+        return self.update_custom_data(id, custom_data)
+
+    def set_validation_schema(self, id: int, schema: Dict[Any, Any]) -> Dict[Any, Any]:
+        """Sets validation schema of the Project by ID.
+        NOTE: This method will overwrite existing validation schema. To extend existing schema,
+        use `get_validation_schema` first to get current schema, then update it and use this method to set new schema.
+
+        :param id: Project ID in Supervisely.
+        :type id: int
+        :param schema: Validation schema to set.
+        :type schema: dict
+        :return: Project information in dict format
+        :rtype: :class:`dict`
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            api = sly.Api.from_env()
+
+            project_id = 123456
+
+            schema = {'key': 'Description of the field'}
+
+            api.project.set_validation_schema(project_id, schema)
+        """
+        return self._edit_validation_schema(id, schema)
+
+    def remove_validation_schema(self, id: int) -> Dict[Any, Any]:
+        """Removes validation schema of the Project by ID.
+
+        :param id: Project ID in Supervisely.
+        :type id: int
+        :return: Project information in dict format
+        :rtype: :class:`dict`
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            api = sly.Api.from_env()
+
+            project_id = 123456
+
+            api.project.remove_validation_schema(project_id)
+        """
+        return self._edit_validation_schema(id)
+
+    def validate_entities_schema(
+        self, id: int, strict: bool = False
+    ) -> List[Dict[str, Union[id, str, List[str], List[Any]]]]:
+        """Validates entities of the Project by ID using validation schema.
+        Returns list of entities that do not match the schema.
+
+        Example of the returned list:
+
+        [
+            {
+                "entity_id": 123456,
+                "entity_name": "image.jpg",
+                "missing_fields": ["location"],
+                "extra_fields": ["city.name"] <- Nested field (field "name" of the field "city")
+            }
+        ]
+
+        :param id: Project ID in Supervisely.
+        :type id: int
+        :param strict: If strict is disabled, only checks if the entity has all the fields from the schema.
+            Any extra fields in the entity will be ignored and will not be considered as an error.
+            If strict is enabled, checks that the entity custom data is an exact match to the schema.
+        :type strict: bool, optional
+        :return: List of dictionaries with information about entities that do not match the schema.
+        :rtype: :class:`List[Dict[str, Union[id, str, List[str], List[Any]]]`
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+
+            api = sly.Api.from_env()
+
+            project_id = 123456
+
+            incorrect_entities = api.project.validate_entities_schema(project_id)
+
+            for entity in incorrect_entities:
+                print(entity.id, entity.name) # Output: 123456, 'image.jpg'
+        """
+        validation_schema = self.get_validation_schema(id)
+        if not validation_schema:
+            raise ValueError("Validation schema is not set for this project.")
+
+        info = self.get_info_by_id(id)
+        listing_method = {
+            # Mapping of project type to listing method.
+            ProjectType.IMAGES.value: self._api.image.get_list,
+        }
+        custom_data_properties = {
+            # Mapping of project type to custom data property name.
+            # Can be different for different project types.
+            ProjectType.IMAGES.value: "meta"
+        }
+
+        listing_method = listing_method.get(info.type)
+        custom_data_property = custom_data_properties.get(info.type)
+
+        if not listing_method or not custom_data_property:
+            # TODO: Add support for other project types.
+            raise NotImplementedError("Validation schema is not supported for this project type.")
+
+        entities = listing_method(project_id=id)
+        incorrect_entities = []
+
+        for entity in entities:
+            custom_data = getattr(entity, custom_data_property)
+            missing_fields, extra_fields = compare_dicts(
+                validation_schema, custom_data, strict=strict
+            )
+            if missing_fields or extra_fields:
+                entry = {
+                    "entity_id": entity.id,
+                    "entity_name": entity.name,
+                    "missing_fields": missing_fields,
+                    "extra_fields": extra_fields,
+                }
+                incorrect_entities.append(entry)
+
+        return incorrect_entities
+
     def get_settings(self, id: int) -> Dict[str, str]:
         info = self._get_info_by_id(id, "projects.info")
         return info.settings

supervisely/api/video/video_annotation_api.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import annotations
 
+import asyncio
 import json
 from typing import Callable, Dict, List, Optional, Union
 
@@ -247,3 +248,47 @@ class VideoAnnotationAPI(EntityAnnotationAPI):
                 self.append(dst_id, ann)
                 if progress_cb is not None:
                     progress_cb(1)
+
+    async def download_async(
+        self,
+        video_id: int,
+        video_info=None,
+        semaphore: Optional[asyncio.Semaphore] = None,
+    ) -> Dict:
+        """
+        Download information about VideoAnnotation by video ID from API asynchronously.
+
+        :param video_id: Video ID in Supervisely.
+        :type video_id: int
+        :param video_info: VideoInfo object. Use it to avoid additional request to the server.
+        :type video_info: VideoInfo, optional
+        :param semaphore: Semaphore to limit the number of parallel downloads.
+        :type semaphore: asyncio.Semaphore, optional
+        :return: Information about VideoAnnotation in json format
+        :rtype: :class:`dict`
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            video_id = 198702499
+            loop = sly.utils.get_or_create_event_loop()
+            ann_info = loop.run_until_complete(api.video.annotation.download_async(video_id))
+        """
+        if video_info is None:
+            video_info = self._api.video.get_info_by_id(video_id)
+
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+
+        async with semaphore:
+            response = await self._api.post_async(
+                self._method_download_bulk,
+                {ApiField.DATASET_ID: video_info.dataset_id, self._entity_ids_str: [video_info.id]},
+            )
+            return response.json()

supervisely/api/video/video_api.py

@@ -2466,8 +2466,7 @@ class VideoApi(RemoveableBulkModuleApi):
             save_path = os.path.join("/path/to/save/", video_info.name)
 
             semaphore = asyncio.Semaphore(100)
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                     api.video.download_path_async(video_info.id, save_path, semaphore)
                 )
@@ -2555,8 +2554,7 @@ class VideoApi(RemoveableBulkModuleApi):
 
             ids = [770914, 770915]
             paths = ["/path/to/save/video1.mp4", "/path/to/save/video2.mp4"]
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(api.video.download_paths_async(ids, paths))
         """
         if len(ids) == 0:

supervisely/api/volume/volume_api.py

@@ -1343,8 +1343,7 @@ class VolumeApi(RemoveableBulkModuleApi):
             save_path = os.path.join("/path/to/save/", volume_info.name)
 
             semaphore = asyncio.Semaphore(100)
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                         api.volume.download_path_async(volume_info.id, save_path, semaphore)
                 )
@@ -1433,8 +1432,7 @@ class VolumeApi(RemoveableBulkModuleApi):
 
             ids = [770914, 770915]
             paths = ["/path/to/save/volume1.nrrd", "/path/to/save/volume2.nrrd"]
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(api.volume.download_paths_async(ids, paths))
         """
         if len(ids) == 0:

supervisely/convert/base_converter.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Optional, Tuple, Union
 
 from tqdm import tqdm
 
-from supervisely._utils import batched, is_production
+from supervisely._utils import batched, get_or_create_event_loop, is_production
 from supervisely.annotation.annotation import Annotation
 from supervisely.annotation.tag_meta import TagValueType
 from supervisely.api.api import Api
@@ -468,7 +468,7 @@ class BaseConverter:
                     for remote_path in files.values()
                 )
 
-            loop = asyncio.get_event_loop()
+            loop = get_or_create_event_loop()
             _, progress_cb = self.get_progress(
                 len(files) if not is_archive_type else file_size,
                 f"Downloading {files_type} from remote storage",
@@ -479,15 +479,19 @@ class BaseConverter:
                 silent_remove(local_path)
 
             logger.info(f"Downloading {files_type} from remote storage...")
-            loop.run_until_complete(
-                self._api.storage.download_bulk_async(
-                    team_id=self._team_id,
-                    remote_paths=list(files.values()),
-                    local_save_paths=list(files.keys()),
-                    progress_cb=progress_cb,
-                    progress_cb_type="number" if not is_archive_type else "size",
-                )
+            download_coro = self._api.storage.download_bulk_async(
+                team_id=self._team_id,
+                remote_paths=list(files.values()),
+                local_save_paths=list(files.keys()),
+                progress_cb=progress_cb,
+                progress_cb_type="number" if not is_archive_type else "size",
             )
+
+            if loop.is_running():
+                future = asyncio.run_coroutine_threadsafe(download_coro, loop=loop)
+                future.result()
+            else:
+                loop.run_until_complete(download_coro)
             logger.info("Possible annotations downloaded successfully.")
 
             if is_archive_type:

supervisely/io/fs.py

@@ -1,7 +1,6 @@
 # coding: utf-8
 
 # docs
-import asyncio
 import errno
 import mimetypes
 import os
@@ -16,7 +15,7 @@ import requests
 from requests.structures import CaseInsensitiveDict
 from tqdm import tqdm
 
-from supervisely._utils import get_bytes_hash, get_string_hash
+from supervisely._utils import get_bytes_hash, get_or_create_event_loop, get_string_hash
 from supervisely.io.fs_cache import FileCache
 from supervisely.sly_logger import logger
 from supervisely.task.progress import Progress
@@ -1375,8 +1374,15 @@ async def copy_file_async(
 
      .. code-block:: python
 
-        from supervisely.io.fs import async_copy_file
-        await async_copy_file('/home/admin/work/projects/example/1.png', '/home/admin/work/tests/2.png')
+        import supervisely as sly
+
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.copy_file_async('/home/admin/work/projects/example/1.png', '/home/admin/work/tests/2.png')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
     """
     ensure_base_path(dst)
     async with aiofiles.open(dst, "wb") as out_f:
@@ -1404,8 +1410,15 @@ async def get_file_hash_async(path: str) -> str:
 
      .. code-block:: python
 
-        from supervisely.io.fs import get_file_hash_async
-        hash = await get_file_hash_async('/home/admin/work/projects/examples/1.jpeg') # rKLYA/p/P64dzidaQ/G7itxIz3ZCVnyUhEE9fSMGxU4=
+        import supervisely as sly
+
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.get_file_hash_async('/home/admin/work/projects/examples/1.jpeg')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            hash = future.result()
+        else:
+            hash = loop.run_until_complete(coro)
     """
     async with aiofiles.open(path, "rb") as file:
         file_bytes = await file.read()
@@ -1442,7 +1455,13 @@ async def unpack_archive_async(
         archive_path = '/home/admin/work/examples.tar'
         target_dir = '/home/admin/work/projects'
 
-        await sly.fs.unpack_archive(archive_path, target_dir)
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.unpack_archive_async(archive_path, target_dir)
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
     """
     if is_split:
         chunk = chunk_size_mb * 1024 * 1024
@@ -1467,9 +1486,37 @@ async def unpack_archive_async(
                         await output_file.write(data)
         archive_path = combined
 
-    loop = asyncio.get_running_loop()
+    loop = get_or_create_event_loop()
     await loop.run_in_executor(None, shutil.unpack_archive, archive_path, target_dir)
     if is_split:
         silent_remove(archive_path)
     if remove_junk:
         remove_junk_from_dir(target_dir)
+
+
+async def touch_async(path: str) -> None:
+    """
+    Sets access and modification times for a file asynchronously.
+
+    :param path: Target file path.
+    :type path: str
+    :returns: None
+    :rtype: :class:`NoneType`
+    :Usage example:
+
+     .. code-block:: python
+
+        import supervisely as sly
+
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.fs.touch_async('/home/admin/work/projects/examples/1.jpeg')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
+    """
+    ensure_base_path(path)
+    async with aiofiles.open(path, "a"):
+        loop = get_or_create_event_loop()
+        await loop.run_in_executor(None, os.utime, path, None)

supervisely/io/json.py

@@ -3,6 +3,7 @@ import json
 import os
 from typing import Dict, Optional
 
+import aiofiles
 import jsonschema
 
 
@@ -230,3 +231,34 @@ def validate_json(data: Dict, schema: Dict, raise_error: bool = False) -> bool:
         if raise_error:
             raise ValueError("JSON data is invalid. See error message for more details.") from err
         return False
+
+
+async def dump_json_file_async(data: Dict, filename: str, indent: Optional[int] = 4) -> None:
+    """
+    Write given data in json format in file with given name asynchronously.
+
+    :param data: Data in json format as a dict.
+    :type data: dict
+    :param filename: Target file path to write data.
+    :type filename: str
+    :param indent: Json array elements and object members will be pretty-printed with that indent level.
+    :type indent: int, optional
+    :returns: None
+    :rtype: :class:`NoneType`
+    :Usage example:
+
+     .. code-block:: python
+
+        import supervisely as sly
+
+        data = {1: 'example'}
+        loop = sly.utils.get_or_create_event_loop()
+        coro = sly.json.dump_json_file_async(data, '/home/admin/work/projects/examples/1.json')
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
+    """
+    async with aiofiles.open(filename, "w") as fout:
+        await fout.write(json.dumps(data, indent=indent))