supervisely 6.73.228__tar.gz → 6.73.229__tar.gz

{supervisely-6.73.228/supervisely.egg-info → supervisely-6.73.229}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.228
+Version: 6.73.229
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/annotation_api.py

@@ -1388,7 +1388,7 @@ class AnnotationApi(ModuleApi):
             ann_info = loop.run_until_complete(api.annotation.download_async(image_id))
         """
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             response = await self._api.post_async(
                 "annotations.info",
@@ -1501,7 +1501,7 @@ class AnnotationApi(ModuleApi):
             context["project_meta"] = project_meta
 
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for image in image_ids:
             task = self.download_async(

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/api.py

@@ -377,6 +377,7 @@ class Api:
 
         self.async_httpx_client: httpx.AsyncClient = None
         self.httpx_client: httpx.Client = None
+        self._semaphore = None
 
     @classmethod
     def normalize_server_address(cls, server_address: str) -> str:
@@ -999,7 +1000,10 @@ class Api:
     def post_httpx(
         self,
         method: str,
-        data: Union[bytes, Dict],
+        json: Dict = None,
+        content: Union[str, bytes, Iterable[bytes], AsyncIterable[bytes]] = None,
+        files: Union[Mapping] = None,
+        params: Union[str, bytes] = None,
         headers: Optional[Dict[str, str]] = None,
         retries: Optional[int] = None,
         raise_error: Optional[bool] = False,
@@ -1009,8 +1013,14 @@ class Api:
 
         :param method: Method name.
         :type method: str
-        :param data: Bytes with data content or dictionary with params.
-        :type data: bytes or dict
+        :param json: Dictionary to send in the body of request.
+        :type json: dict, optional
+        :param content: Bytes with data content or dictionary with params.
+        :type content: bytes or dict, optional
+        :param files: Files to send in the body of request.
+        :type files: dict, optional
+        :param params: URL query parameters.
+        :type params: str, bytes, optional
         :param headers: Custom headers to include in the request.
         :type headers: dict, optional
         :param retries: The number of attempts to connect to the server.
@@ -1021,6 +1031,7 @@ class Api:
         :rtype: :class:`httpx.Response`
         """
         self._set_client()
+
         if retries is None:
             retries = self.retry_count
 
@@ -1032,18 +1043,17 @@ class Api:
         else:
             headers = {**self.headers, **headers}
 
-        if isinstance(data, bytes):
-            request_params = {"content": data}
-        elif isinstance(data, Dict):
-            json_body = {**data, **self.additional_fields}
-            request_params = {"json": json_body}
-        else:
-            request_params = {"params": data}
-
         for retry_idx in range(retries):
             response = None
             try:
-                response = self.httpx_client.post(url, headers=headers, **request_params)
+                response = self.httpx_client.post(
+                    url,
+                    content=content,
+                    files=files,
+                    json=json,
+                    params=params,
+                    headers=headers,
+                )
                 if response.status_code != httpx.codes.OK:
                     self._check_version()
                     Api._raise_for_status_httpx(response)
@@ -1065,9 +1075,8 @@ class Api:
                     )
             except Exception as exc:
                 process_unhandled_request(self.logger, exc)
-        raise httpx.HTTPStatusError(
-            "Retry limit exceeded ({!r})".format(url),
-            response=response,
+        raise httpx.RequestError(
+            f"Retry limit exceeded ({url})",
             request=getattr(response, "request", None),
         )
 
@@ -1093,6 +1102,7 @@ class Api:
         :rtype: :class:`Response<Response>`
         """
         self._set_client()
+
         if retries is None:
             retries = self.retry_count
 
@@ -1171,8 +1181,8 @@ class Api:
         :return: Generator object.
         :rtype: :class:`Generator`
         """
-
         self._set_client()
+
         if retries is None:
             retries = self.retry_count
 
@@ -1408,14 +1418,13 @@ class Api:
         url = self.api_server_address + "/v3/" + method
         if not use_public_api:
             url = os.path.join(self.server_address, method)
+        logger.trace(f"{method_type} {url}")
 
         if headers is None:
             headers = self.headers.copy()
         else:
             headers = {**self.headers, **headers}
 
-        logger.trace(f"{method_type} {url}")
-
         if isinstance(data, (bytes, Generator)):
             content = data
             json_body = None
@@ -1506,42 +1515,80 @@ class Api:
 
     def _set_async_client(self):
         """
-        Set async httpx client if it is not set yet.
-        Switch on HTTP/2 if the server address uses HTTPS.
+        Set async httpx client with HTTP/2 if it is not set yet.
         """
         if self.async_httpx_client is None:
-            self._check_https_redirect()
-            if self.server_address.startswith("https://"):
-                self.async_httpx_client = httpx.AsyncClient(http2=True)
-            else:
-                self.async_httpx_client = httpx.AsyncClient()
+            self.async_httpx_client = httpx.AsyncClient(http2=True)
 
     def _set_client(self):
         """
-        Set httpx client if it is not set yet.
-        Switch on HTTP/2 if the server address uses HTTPS.
+        Set sync httpx client with HTTP/2 if it is not set yet.
         """
         if self.httpx_client is None:
-            self._check_https_redirect()
-            if self.server_address.startswith("https://"):
-                self.httpx_client = httpx.Client(http2=True)
-            else:
-                self.httpx_client = httpx.Client()
+            self.httpx_client = httpx.Client(http2=True)
+
+    def get_default_semaphore(self) -> asyncio.Semaphore:
+        """
+        Get default global API semaphore for async requests.
+        If the semaphore is not set, it will be initialized.
+
+        During initialization, the semaphore size will be set from the environment variable SUPERVISELY_ASYNC_SEMAPHORE.
+        If the environment variable is not set, the default value will be set based on the server address.
+        Depending on the server address, the semaphore size will be set to 10 for HTTPS and 5 for HTTP.
+
+        :return: Semaphore object.
+        :rtype: :class:`asyncio.Semaphore`
+        """
+        if self._semaphore is None:
+            self._initialize_semaphore()
+        return self._semaphore
 
-    def _get_default_semaphore(self):
+    def _initialize_semaphore(self):
         """
-        Get default semaphore for async requests.
-        Check if the environment variable SUPERVISELY_ASYNC_SEMAPHORE is set.
-        If it is set, create a semaphore with the given value.
-        Otherwise, create a semaphore with a default value.
-        If server supports HTTPS, create a semaphore with a higher value.
+        Initialize the semaphore for async requests.
+
+        If the environment variable SUPERVISELY_ASYNC_SEMAPHORE is set, create a semaphore with the given value.
+
+        Otherwise, create a semaphore with a default value:
+
+            - If server supports HTTPS, create a semaphore with value 10.
+            - If server supports HTTP, create a semaphore with value 5.
         """
-        env_semaphore = os.getenv("SUPERVISELY_ASYNC_SEMAPHORE")
-        if env_semaphore is not None:
-            return asyncio.Semaphore(int(env_semaphore))
+        semaphore_size = sly_env.semaphore_size()
+        if semaphore_size is not None:
+            self._semaphore = asyncio.Semaphore(semaphore_size)
+            logger.debug(
+                f"Setting global API semaphore size to {semaphore_size} from environment variable"
+            )
         else:
             self._check_https_redirect()
             if self.server_address.startswith("https://"):
-                return asyncio.Semaphore(25)
+                self._semaphore = asyncio.Semaphore(10)
+                logger.debug("Setting global API semaphore size to 10 for HTTPS")
             else:
-                return asyncio.Semaphore(5)
+                self._semaphore = asyncio.Semaphore(5)
+                logger.debug("Setting global API semaphore size to 5 for HTTP")
+
+    def set_semaphore_size(self, size: int = None):
+        """
+        Set the global API semaphore with the given size. Will replace the existing semaphore.
+        If the size is not set, will set from the environment variable SUPERVISELY_ASYNC_SEMAPHORE.
+        If the environment variable is not set, will set the default value.
+
+        :param size: Size of the semaphore.
+        :type size: int, optional
+        """
+        if size is not None:
+            self._semaphore = asyncio.Semaphore(size)
+        else:
+            self._initialize_semaphore()
+
+    @property
+    def semaphore(self) -> asyncio.Semaphore:
+        """
+        Get the global API semaphore for async requests.
+
+        :return: Semaphore object.
+        :rtype: :class:`asyncio.Semaphore`
+        """
+        return self._semaphore

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/app_api.py

@@ -551,7 +551,7 @@ class AppApi(TaskApi):
 
         def is_enabled(self) -> bool:
             """Check if the workflow functionality is enabled."""
-            logger.info(f"Workflow check: is {'enabled' if self._enabled else 'disabled'}.")
+            logger.debug(f"Workflow check: is {'enabled' if self._enabled else 'disabled'}.")
             return self._enabled
 
         # pylint: disable=no-self-argument
@@ -579,8 +579,8 @@ class AppApi(TaskApi):
                         ):
                             self.__last_warning_time = time.monotonic()
                             logger.warning(
-                                "Workflow is disabled in development mode. "
-                                "To enable it, use `api.app.workflow.enable()` right after Api initialization."
+                                "Workflow is disabled. "
+                                "To enable it, use `api.app.workflow.enable()`."
                             )
                         return
                     if not check_workflow_compatibility(self._api, version_to_check):

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/file_api.py

@@ -1646,7 +1646,7 @@ class FileApi(ModuleApiBase):
             loop.run_until_complete(api.file.download_async(8, path_to_file, local_save_path))
         """
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             if self.is_on_agent(remote_path):
                 # for optimized download from agent
@@ -1774,7 +1774,7 @@ class FileApi(ModuleApiBase):
             )
 
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
 
         tasks = []
         for remote_path, local_path, cache in zip(
@@ -1836,7 +1836,7 @@ class FileApi(ModuleApiBase):
         """
 
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
 
         if not remote_path.endswith("/"):
             remote_path += "/"
@@ -1925,7 +1925,7 @@ class FileApi(ModuleApiBase):
         """
 
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
 
         remote_file_path = env.file(raise_not_found=False)
         remote_folder_path = env.folder(raise_not_found=False)
@@ -2061,7 +2061,7 @@ class FileApi(ModuleApiBase):
             # "sha256": sha256, #TODO add with resumaple api
         }
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             async with aiofiles.open(src, "rb") as fd:
                 item = await fd.read()
@@ -2129,7 +2129,7 @@ class FileApi(ModuleApiBase):
                 )
         """
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for s, d in zip(src_paths, dst_paths):
             task = self.upload_async(

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/image_api.py

@@ -3639,7 +3639,7 @@ class ImageApi(RemoveableBulkModuleApi):
             results = await asyncio.gather(*tasks)
         """
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
 
         async with semaphore:
             async for response in self._download_async(id):
@@ -3697,7 +3697,7 @@ class ImageApi(RemoveableBulkModuleApi):
 
         """
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = [
             self.download_np_async(id, semaphore, keep_alpha, progress_cb, progress_cb_type)
             for id in ids
@@ -3772,7 +3772,7 @@ class ImageApi(RemoveableBulkModuleApi):
         ensure_base_path(path)
         hash_to_check = None
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             async with aiofiles.open(path, writing_method) as fd:
                 async for chunk, hhash in self._download_async(
@@ -3850,7 +3850,7 @@ class ImageApi(RemoveableBulkModuleApi):
                 f'Length of "ids" and "paths" should be equal. {len(ids)} != {len(paths)}'
             )
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for img_id, img_path in zip(ids, paths):
             task = self.download_path_async(
@@ -3924,7 +3924,7 @@ class ImageApi(RemoveableBulkModuleApi):
         hash_to_check = None
 
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             content = b""
             async for chunk, hhash in self._download_async(
@@ -3993,7 +3993,7 @@ class ImageApi(RemoveableBulkModuleApi):
                 img_bytes_list = loop.run_until_complete(api.image.download_bytes_imgs_async(ids, semaphore))
         """
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for id in ids:
             task = self.download_bytes_single_async(

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/pointcloud/pointcloud_api.py

@@ -1166,7 +1166,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
         ensure_base_path(path)
         hash_to_check = None
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             async with aiofiles.open(path, writing_method) as fd:
                 async for chunk, hhash in self._download_async(
@@ -1248,7 +1248,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
         if len(ids) != len(paths):
             raise ValueError('Can not match "ids" and "paths" lists, len(ids) != len(paths)')
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for img_id, img_path in zip(ids, paths):
             task = self.download_path_async(
@@ -1322,7 +1322,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
         ensure_base_path(path)
         hash_to_check = None
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             async with aiofiles.open(path, "wb") as fd:
                 async for chunk, hhash in self._api.stream_async(
@@ -1403,7 +1403,7 @@ class PointcloudApi(RemoveableBulkModuleApi):
         if len(ids) != len(paths):
             raise ValueError('Can not match "ids" and "paths" lists, len(ids) != len(paths)')
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for img_id, img_path in zip(ids, paths):
             task = self.download_related_image_async(

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/video/video_annotation_api.py

@@ -284,7 +284,7 @@ class VideoAnnotationAPI(EntityAnnotationAPI):
             video_info = self._api.video.get_info_by_id(video_id)
 
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
 
         async with semaphore:
             response = await self._api.post_async(

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/video/video_api.py

@@ -2482,7 +2482,7 @@ class VideoApi(RemoveableBulkModuleApi):
         ensure_base_path(path)
         hash_to_check = None
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             async with aiofiles.open(path, writing_method) as fd:
                 async for chunk, hhash in self._download_async(
@@ -2562,7 +2562,7 @@ class VideoApi(RemoveableBulkModuleApi):
         if len(ids) != len(paths):
             raise ValueError('Can not match "ids" and "paths" lists, len(ids) != len(paths)')
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for img_id, img_path in zip(ids, paths):
             task = self.download_path_async(

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/api/volume/volume_api.py

@@ -1360,7 +1360,7 @@ class VolumeApi(RemoveableBulkModuleApi):
         ensure_base_path(path)
         hash_to_check = None
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         async with semaphore:
             async with aiofiles.open(path, writing_method) as fd:
                 async for chunk, hhash in self._download_async(
@@ -1440,7 +1440,7 @@ class VolumeApi(RemoveableBulkModuleApi):
         if len(ids) != len(paths):
             raise ValueError(f'Can not match "ids" and "paths" lists, {len(ids)} != {len(paths)}')
         if semaphore is None:
-            semaphore = self._api._get_default_semaphore()
+            semaphore = self._api.get_default_semaphore()
         tasks = []
         for img_id, img_path in zip(ids, paths):
             task = self.download_path_async(

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/io/env.py

@@ -539,3 +539,18 @@ def mininum_instance_version_for_sdk() -> str:
         postprocess_fn=lambda x: x,
         raise_not_found=False,
     )
+
+
+def semaphore_size() -> int:
+    """Returns semaphore size from environment variable using following
+        - SUPERVISELY_ASYNC_SEMAPHORE
+
+    :return: semaphore size
+    :rtype: int
+    """
+    return _parse_from_env(
+        name="semaphore_size",
+        keys=["SUPERVISELY_ASYNC_SEMAPHORE"],
+        postprocess_fn=lambda x: int(x),
+        raise_not_found=False,
+    )

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/project/project.py

@@ -4387,7 +4387,7 @@ async def _download_project_async(
     resume_download: Optional[bool] = False,
 ):
     if semaphore is None:
-        semaphore = api._get_default_semaphore()
+        semaphore = api.get_default_semaphore()
 
     dataset_ids = set(dataset_ids) if (dataset_ids is not None) else None
     project_fs = None

{supervisely-6.73.228 → supervisely-6.73.229}/supervisely/project/video_project.py

@@ -1594,7 +1594,7 @@ async def download_video_project_async(
                     )
     """
     if semaphore is None:
-        semaphore = api._get_default_semaphore()
+        semaphore = api.get_default_semaphore()
 
     key_id_map = KeyIdMap()
 

{supervisely-6.73.228 → supervisely-6.73.229/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.228
+Version: 6.73.229
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.228 → supervisely-6.73.229}/tests/test_async_image_api.py

@@ -8,8 +8,8 @@ from tqdm.asyncio import tqdm
 
 import supervisely as sly
 
-LOG_LEVEL = "INFO"
-# LOG_LEVEL = "DEBUG"
+# LOG_LEVEL = "INFO"
+LOG_LEVEL = "DEBUG"
 DATASET_ID = 98357
 user_path = os.path.expanduser("~")
 save_path = f"{user_path}/Work/test_images_download/"
@@ -113,8 +113,8 @@ if __name__ == "__main__":
     try:
         # main_dnp()  # to download and save images as numpy arrays
         # main_dp()  # to download and save images as files
-        # main_dps()  # to download and save images as files (batch)
-        compare_main_dps()  # to compare the time taken for downloading images as files (batch)
+        main_dps()  # to download and save images as files (batch)
+        # compare_main_dps()  # to compare the time taken for downloading images as files (batch)
         # main_bytes()  # to download and save image as bytes
         # main_n_bytes()  # to download and save images as bytes (batch)
     except KeyboardInterrupt: