supervisely 6.73.226__py3-none-any.whl → 6.73.228__py3-none-any.whl

supervisely/api/file_api.py

@@ -14,6 +14,7 @@ from time import time
 from typing import Callable, Dict, List, NamedTuple, Optional, Union
 
 import aiofiles
+import httpx
 import requests
 from dotenv import load_dotenv
 from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
@@ -1641,8 +1642,7 @@ class FileApi(ModuleApiBase):
 
             path_to_file = "/999_App_Test/ds1/01587.json"
             local_save_path = "/path/to/save/999_App_Test/ds1/01587.json"
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(api.file.download_async(8, path_to_file, local_save_path))
         """
         if semaphore is None:
@@ -1756,8 +1756,7 @@ class FileApi(ModuleApiBase):
                 "/path/to/save/999_App_Test/ds1/01588.json",
                 "/path/to/save/999_App_Test/ds1/01587.json"
             ]
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                     api.file.download_bulk_async(8, paths_to_files, local_paths)
                 )
@@ -1830,8 +1829,7 @@ class FileApi(ModuleApiBase):
             path_to_dir = "/files/folder"
             local_path = "path/to/local/folder"
 
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                     api.file.download_directory_async(9, path_to_dir, local_path)
                 )
@@ -1918,8 +1916,7 @@ class FileApi(ModuleApiBase):
 
             # Application is started...
             save_path = "/my_app_data"
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                     api.file.download_input_async(save_path)
                 )
@@ -2008,3 +2005,142 @@ class FileApi(ModuleApiBase):
                 semaphore=semaphore,
                 show_progress=show_progress,
             )
+
+    async def upload_async(
+        self,
+        team_id: int,
+        src: str,
+        dst: str,
+        semaphore: Optional[asyncio.Semaphore] = None,
+        # chunk_size: int = 1024 * 1024, #TODO add with resumaple api
+        # check_hash: bool = True, #TODO add with resumaple api
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "size",
+    ) -> httpx.Response:
+        """
+        Upload file from local path to Team Files asynchronously.
+
+        :param team_id: Team ID in Supervisely.
+        :type team_id: int
+        :param src: Local path to file.
+        :type src: str
+        :param dst: Path to save file in Team Files.
+        :type dst: str
+        :param semaphore: Semaphore for limiting the number of simultaneous uploads.
+        :type semaphore: asyncio.Semaphore, optional
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: tqdm or callable, optional
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
+        :type progress_cb_type: Literal["number", "size"], optional
+        :return: Response from API.
+        :rtype: :class:`httpx.Response`
+        :Usage example:
+
+            .. code-block:: python
+
+                import supervisely as sly
+                import asyncio
+
+                os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+                os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+                api = sly.Api.from_env()
+
+                path_to_file = "/path/to/local/file/01587.json"
+                path_to_save = "/files/01587.json"
+                loop = sly.utils.get_or_create_event_loop()
+                loop.run_until_complete(
+                    api.file.upload_async(8, path_to_file, path_to_save)
+                )
+        """
+        api_method = "file-storage.upload"
+        headers = {"Content-Type": "application/octet-stream"}
+        # sha256 = await get_file_hash_async(src) #TODO add with resumaple api
+        json_body = {
+            ApiField.TEAM_ID: team_id,
+            ApiField.PATH: dst,
+            # "sha256": sha256, #TODO add with resumaple api
+        }
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+        async with semaphore:
+            async with aiofiles.open(src, "rb") as fd:
+                item = await fd.read()
+                response = await self._api.post_async(
+                    api_method, content=item, params=json_body, headers=headers
+                )
+                if progress_cb is not None and progress_cb_type == "size":
+                    progress_cb(len(item))
+                if progress_cb is not None and progress_cb_type == "number":
+                    progress_cb(1)
+                return response
+
+    async def upload_bulk_async(
+        self,
+        team_id: int,
+        src_paths: List[str],
+        dst_paths: List[str],
+        semaphore: Optional[asyncio.Semaphore] = None,
+        # chunk_size: int = 1024 * 1024, #TODO add with resumaple api
+        # check_hash: bool = True, #TODO add with resumaple api
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "size",
+    ) -> None:
+        """
+        Upload multiple files from local paths to Team Files asynchronously.
+
+        :param team_id: Team ID in Supervisely.
+        :type team_id: int
+        :param src_paths: List of local paths to files.
+        :type src_paths: List[str]
+        :param dst_paths: List of paths to save files in Team Files.
+        :type dst_paths: List[str]
+        :param semaphore: Semaphore for limiting the number of simultaneous uploads.
+        :type semaphore: asyncio.Semaphore, optional
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: tqdm or callable, optional
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
+        :type progress_cb_type: Literal["number", "size"], optional
+        :return: None
+        :rtype: :class:`NoneType`
+        :Usage example:
+
+            .. code-block:: python
+
+                import supervisely as sly
+                import asyncio
+
+                os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+                os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+                api = sly.Api.from_env()
+
+                paths_to_files = [
+                    "/path/to/local/file/01587.json",
+                    "/path/to/local/file/01588.json",
+                    "/path/to/local/file/01589.json"
+                ]
+                paths_to_save = [
+                    "/files/01587.json",
+                    "/files/01588.json",
+                    "/files/01589.json"
+                ]
+                loop = sly.utils.get_or_create_event_loop()
+                loop.run_until_complete(
+                    api.file.upload_bulk_async(8, paths_to_files, paths_to_save)
+                )
+        """
+        if semaphore is None:
+            semaphore = self._api._get_default_semaphore()
+        tasks = []
+        for s, d in zip(src_paths, dst_paths):
+            task = self.upload_async(
+                team_id,
+                s,
+                d,
+                semaphore=semaphore,
+                # chunk_size=chunk_size, #TODO add with resumaple api
+                # check_hash=check_hash, #TODO add with resumaple api
+                progress_cb=progress_cb,
+                progress_cb_type=progress_cb_type,
+            )
+            tasks.append(task)
+        await asyncio.gather(*tasks)

supervisely/api/image_api.py

@@ -40,6 +40,7 @@ from tqdm import tqdm
 
 from supervisely._utils import (
     batched,
+    compare_dicts,
     generate_free_name,
     get_bytes_hash,
     resize_image_url,
@@ -1135,7 +1136,14 @@ class ImageApi(RemoveableBulkModuleApi):
         )
 
     def upload_path(
-        self, dataset_id: int, name: str, path: str, meta: Optional[Dict] = None
+        self,
+        dataset_id: int,
+        name: str,
+        path: str,
+        meta: Optional[Dict] = None,
+        validate_meta: Optional[bool] = False,
+        use_strict_validation: Optional[bool] = False,
+        use_caching_for_validation: Optional[bool] = False,
     ) -> ImageInfo:
         """
         Uploads Image with given name from given local path to Dataset.
@@ -1148,6 +1156,12 @@ class ImageApi(RemoveableBulkModuleApi):
         :type path: str
         :param meta: Image metadata.
         :type meta: dict, optional
+        :param validate_meta: If True, validates provided meta with saved JSON schema.
+        :type validate_meta: bool, optional
+        :param use_strict_validation: If True, uses strict validation.
+        :type use_strict_validation: bool, optional
+        :param use_caching_for_validation: If True, uses caching for validation.
+        :type use_caching_for_validation: bool, optional
         :return: Information about Image. See :class:`info_sequence<info_sequence>`
         :rtype: :class:`ImageInfo`
         :Usage example:
@@ -1163,7 +1177,15 @@ class ImageApi(RemoveableBulkModuleApi):
             img_info = api.image.upload_path(dataset_id, name="7777.jpeg", path="/home/admin/Downloads/7777.jpeg")
         """
         metas = None if meta is None else [meta]
-        return self.upload_paths(dataset_id, [name], [path], metas=metas)[0]
+        return self.upload_paths(
+            dataset_id,
+            [name],
+            [path],
+            metas=metas,
+            validate_meta=validate_meta,
+            use_strict_validation=use_strict_validation,
+            use_caching_for_validation=use_caching_for_validation,
+        )[0]
 
     def upload_paths(
         self,
@@ -1173,6 +1195,9 @@ class ImageApi(RemoveableBulkModuleApi):
         progress_cb: Optional[Union[tqdm, Callable]] = None,
         metas: Optional[List[Dict]] = None,
         conflict_resolution: Optional[Literal["rename", "skip", "replace"]] = None,
+        validate_meta: Optional[bool] = False,
+        use_strict_validation: Optional[bool] = False,
+        use_caching_for_validation: Optional[bool] = False,
     ) -> List[ImageInfo]:
         """
         Uploads Images with given names from given local path to Dataset.
@@ -1189,6 +1214,12 @@ class ImageApi(RemoveableBulkModuleApi):
         :type metas: List[dict], optional
         :param conflict_resolution: The strategy to resolve upload conflicts. 'Replace' option will replace the existing images in the dataset with the new images. The images that are being deleted are logged. 'Skip' option will ignore the upload of new images that would result in a conflict. An original image's ImageInfo list will be returned instead. 'Rename' option will rename the new images to prevent any conflict.
         :type conflict_resolution: Optional[Literal["rename", "skip", "replace"]]
+        :param validate_meta: If True, validates provided meta with saved JSON schema.
+        :type validate_meta: bool, optional
+        :param use_strict_validation: If True, uses strict validation.
+        :type use_strict_validation: bool, optional
+        :param use_caching_for_validation: If True, uses caching for validation.
+        :type use_caching_for_validation: bool, optional
         :raises: :class:`ValueError` if len(names) != len(paths)
         :return: List with information about Images. See :class:`info_sequence<info_sequence>`
         :rtype: :class:`List[ImageInfo]`
@@ -1214,7 +1245,14 @@ class ImageApi(RemoveableBulkModuleApi):
         self._upload_data_bulk(path_to_bytes_stream, zip(paths, hashes), progress_cb=progress_cb)
 
         return self.upload_hashes(
-            dataset_id, names, hashes, metas=metas, conflict_resolution=conflict_resolution
+            dataset_id,
+            names,
+            hashes,
+            metas=metas,
+            conflict_resolution=conflict_resolution,
+            validate_meta=validate_meta,
+            use_strict_validation=use_strict_validation,
+            use_caching_for_validation=use_caching_for_validation,
         )
 
     def upload_np(
@@ -1492,6 +1530,9 @@ class ImageApi(RemoveableBulkModuleApi):
         batch_size: Optional[int] = 50,
         skip_validation: Optional[bool] = False,
         conflict_resolution: Optional[Literal["rename", "skip", "replace"]] = None,
+        validate_meta: Optional[bool] = False,
+        use_strict_validation: Optional[bool] = False,
+        use_caching_for_validation: Optional[bool] = False,
     ) -> List[ImageInfo]:
         """
         Upload images from given hashes to Dataset.
@@ -1512,6 +1553,12 @@ class ImageApi(RemoveableBulkModuleApi):
         :type skip_validation: bool, optional
         :param conflict_resolution: The strategy to resolve upload conflicts. 'Replace' option will replace the existing images in the dataset with the new images. The images that are being deleted are logged. 'Skip' option will ignore the upload of new images that would result in a conflict. An original image's ImageInfo list will be returned instead. 'Rename' option will rename the new images to prevent any conflict.
         :type conflict_resolution: Optional[Literal["rename", "skip", "replace"]]
+        :param validate_meta: If True, validates provided meta with saved JSON schema.
+        :type validate_meta: bool, optional
+        :param use_strict_validation: If True, uses strict validation.
+        :type use_strict_validation: bool, optional
+        :param use_caching_for_validation: If True, uses caching for validation.
+        :type use_caching_for_validation: bool, optional
         :return: List with information about Images. See :class:`info_sequence<info_sequence>`
         :rtype: :class:`List[ImageInfo]`
         :Usage example:
@@ -1553,6 +1600,9 @@ class ImageApi(RemoveableBulkModuleApi):
             batch_size=batch_size,
             skip_validation=skip_validation,
             conflict_resolution=conflict_resolution,
+            validate_meta=validate_meta,
+            use_strict_validation=use_strict_validation,
+            use_caching_for_validation=use_caching_for_validation,
         )
 
     def upload_id(
@@ -1750,8 +1800,44 @@ class ImageApi(RemoveableBulkModuleApi):
         force_metadata_for_links=True,
         skip_validation=False,
         conflict_resolution: Optional[Literal["rename", "skip", "replace"]] = None,
+        validate_meta: Optional[bool] = False,
+        use_strict_validation: Optional[bool] = False,
+        use_caching_for_validation: Optional[bool] = False,
     ):
         """ """
+        if use_strict_validation and not validate_meta:
+            raise ValueError(
+                "use_strict_validation is set to True, while validate_meta is set to False. "
+                "Please set validate_meta to True to use strict validation "
+                "or disable strict validation by setting use_strict_validation to False."
+            )
+        if validate_meta:
+            dataset_info = self._api.dataset.get_info_by_id(dataset_id)
+
+            validation_schema = self._api.project.get_validation_schema(
+                dataset_info.project_id, use_caching=use_caching_for_validation
+            )
+
+            if validation_schema is None:
+                raise ValueError(
+                    "Validation schema is not set for the project, while "
+                    "validate_meta is set to True. Either disable the validation "
+                    "or set the validation schema for the project using the "
+                    "api.project.set_validation_schema method."
+                )
+
+            for idx, meta in enumerate(metas):
+                missing_fields, extra_fields = compare_dicts(
+                    validation_schema, meta, strict=use_strict_validation
+                )
+
+                if missing_fields or extra_fields:
+                    raise ValueError(
+                        f"Validation failed for the metadata of the image with index {idx} and name {names[idx]}. "
+                        "Please check the metadata and try again. "
+                        f"Missing fields: {missing_fields}, Extra fields: {extra_fields}"
+                    )
+
         if (
             conflict_resolution is not None
             and conflict_resolution not in SUPPORTED_CONFLICT_RESOLUTIONS
@@ -3604,8 +3690,7 @@ class ImageApi(RemoveableBulkModuleApi):
                 semaphore = asyncio.Semaphore(100)
                 images = api.image.get_list(DATASET_ID)
                 img_ids = [image.id for image in images]
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
+                loop = sly.utils.get_or_create_event_loop()
                 results = loop.run_until_complete(
                                 api.image.download_nps_async(img_ids, semaphore)
                             )
@@ -3671,8 +3756,7 @@ class ImageApi(RemoveableBulkModuleApi):
             save_path = os.path.join("/path/to/save/", img_info.name)
 
             semaphore = asyncio.Semaphore(100)
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                     api.image.download_path_async(img_info.id, save_path, semaphore)
                 )
@@ -3756,8 +3840,7 @@ class ImageApi(RemoveableBulkModuleApi):
 
             ids = [770918, 770919]
             paths = ["/path/to/save/image1.png", "/path/to/save/image2.png"]
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(api.image.download_paths_async(ids, paths))
         """
         if len(ids) == 0:
@@ -3828,8 +3911,7 @@ class ImageApi(RemoveableBulkModuleApi):
             api = sly.Api.from_env()
 
             img_id = 770918
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             img_bytes = loop.run_until_complete(api.image.download_bytes_async(img_id))
 
         """
@@ -3906,8 +3988,7 @@ class ImageApi(RemoveableBulkModuleApi):
                 os.environ['API_TOKEN
                 api = sly.Api.from_env()
 
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
+                loop = sly.utils.get_or_create_event_loop()
                 semaphore = asyncio.Semaphore(100)
                 img_bytes_list = loop.run_until_complete(api.image.download_bytes_imgs_async(ids, semaphore))
         """

supervisely/api/pointcloud/pointcloud_api.py

@@ -1149,8 +1149,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
             save_path = os.path.join("/path/to/save/", pcd_info.name)
 
             semaphore = asyncio.Semaphore(100)
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                         api.pointcloud.download_path_async(pcd_info.id, save_path, semaphore)
                     )
@@ -1241,8 +1240,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
 
             ids = [19373403, 19373404]
             paths = ["/path/to/save/000063.pcd", "/path/to/save/000064.pcd"]
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(api.pointcloud.download_paths_async(ids, paths))
         """
         if len(ids) == 0:
@@ -1313,8 +1311,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
             save_path = os.path.join("/path/to/save/", img_info.name)
 
             semaphore = asyncio.Semaphore(100)
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
+            loop = sly.utils.get_or_create_event_loop()
             loop.run_until_complete(
                     api.pointcloud.download_related_image_async(19373403, save_path, semaphore)
                 )
@@ -1395,8 +1392,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
                 save_paths = [os.path.join("/path/to/save/", img_info.name) for img_info in img_infos]
 
                 semaphore = asyncio.Semaphore(100)
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
+                loop = sly.utils.get_or_create_event_loop()
                 loop.run_until_complete(
                         api.pointcloud.download_related_images_async(ids, save_paths, semaphore)
                     )