supervisely 6.73.323__tar.gz → 6.73.324__tar.gz

{supervisely-6.73.323/supervisely.egg-info → supervisely-6.73.324}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.323
+Version: 6.73.324
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.323 → supervisely-6.73.324}/supervisely/_utils.py

@@ -471,9 +471,20 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
             return loop
 
 
-def sync_call(coro):
+def run_coroutine(coroutine):
     """
-    This function is used to run asynchronous functions in synchronous context.
+    Runs an asynchronous coroutine in a synchronous context and waits for its result.
+
+    This function checks if an event loop is already running:
+    - If a loop is running, it schedules the coroutine using `asyncio.run_coroutine_threadsafe()`
+      and waits for the result.
+    - If no loop is running, it creates one and executes the coroutine with `run_until_complete()`.
+
+    This ensures compatibility with both synchronous and asynchronous environments
+    without creating unnecessary event loops.
+
+    ⚠️ Note: This method is preferable when working with `asyncio` objects like `Semaphore`,
+    since it avoids issues with mismatched event loops.
 
     :param coro: Asynchronous function.
     :type coro: Coroutine
@@ -484,13 +495,13 @@ def sync_call(coro):
 
     .. code-block:: python
 
-            from supervisely.utils import sync_call
+            from supervisely._utils import run_coroutine
 
             async def async_function():
                 await asyncio.sleep(1)
                 return "Hello, World!"
             coro = async_function()
-            result = sync_call(coro)
+            result = run_coroutine(coro)
             print(result)
             # Output: Hello, World!
     """
@@ -498,10 +509,10 @@ def sync_call(coro):
     loop = get_or_create_event_loop()
 
     if loop.is_running():
-        future = asyncio.run_coroutine_threadsafe(coro, loop=loop)
+        future = asyncio.run_coroutine_threadsafe(coroutine, loop=loop)
         return future.result()
     else:
-        return loop.run_until_complete(coro)
+        return loop.run_until_complete(coroutine)
 
 
 def get_filename_from_headers(url):

{supervisely-6.73.323 → supervisely-6.73.324}/supervisely/api/file_api.py

@@ -23,7 +23,7 @@ from typing_extensions import Literal
 
 import supervisely.io.env as env
 import supervisely.io.fs as sly_fs
-from supervisely._utils import batched, rand_str
+from supervisely._utils import batched, rand_str, run_coroutine
 from supervisely.api.module_api import ApiField, ModuleApiBase
 from supervisely.io.fs import (
     ensure_base_path,
@@ -2302,3 +2302,89 @@ class FileApi(ModuleApiBase):
             else:
                 raise e
         return res_remote_dir
+
+    def upload_directory_fast(
+        self,
+        team_id: int,
+        local_dir: str,
+        remote_dir: str,
+        change_name_if_conflict: Optional[bool] = True,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        replace_if_conflict: Optional[bool] = False,
+        enable_fallback: Optional[bool] = True,
+    ) -> str:
+        """
+        Upload Directory to Team Files from local path in fast mode.
+        Files are uploaded asynchronously. If an error occurs, the method will fallback to synchronous upload.
+
+        :param team_id: Team ID in Supervisely.
+        :type team_id: int
+        :param local_dir: Path to local Directory.
+        :type local_dir: str
+        :param remote_dir: Path to Directory in Team Files.
+        :type remote_dir: str
+        :param change_name_if_conflict: Checks if given name already exists and adds suffix to the end of the name.
+        :type change_name_if_conflict: bool, optional
+        :param progress_cb: Function for tracking download progress in bytes.
+        :type progress_cb: Progress, optional
+        :param replace_if_conflict: If True, replace existing dir.
+        :type replace_if_conflict: bool, optional
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
+        :return: Path to Directory in Team Files
+        :rtype: :class:`str`
+        """
+        coroutine = self.upload_directory_async(
+            team_id=team_id,
+            local_dir=local_dir,
+            remote_dir=remote_dir,
+            change_name_if_conflict=change_name_if_conflict,
+            progress_size_cb=progress_cb,
+            replace_if_conflict=replace_if_conflict,
+            enable_fallback=enable_fallback,
+        )
+        return run_coroutine(coroutine)
+
+    def upload_bulk_fast(
+        self,
+        team_id: int,
+        src_paths: List[str],
+        dst_paths: List[str],
+        semaphore: Optional[asyncio.Semaphore] = None,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        progress_cb_type: Literal["number", "size"] = "size",
+        enable_fallback: Optional[bool] = True,
+    ) -> None:
+        """
+        Upload multiple files from local paths to Team Files in fast mode.
+        Files are uploaded asynchronously. If an error occurs, the method will fallback to synchronous upload.
+
+        :param team_id: Team ID in Supervisely.
+        :type team_id: int
+        :param src_paths: List of local paths to files.
+        :type src_paths: List[str]
+        :param dst_paths: List of paths to save files in Team Files.
+        :type dst_paths: List[str]
+        :param semaphore: Semaphore for limiting the number of simultaneous uploads.
+        :type semaphore: asyncio.Semaphore, optional
+        :param progress_cb: Function for tracking download progress.
+        :type progress_cb: tqdm or callable, optional
+        :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
+                                "size" is used to track the number of transferred bytes.
+                                "number" is used to track the number of transferred files.
+        :type progress_cb_type: Literal["number", "size"], optional 
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
+        :return: None
+        :rtype: :class:`NoneType`
+        """
+        coroutine = self.upload_bulk_async(
+            team_id=team_id,
+            src_paths=src_paths,
+            dst_paths=dst_paths,
+            semaphore=semaphore,
+            progress_cb=progress_cb,
+            progress_cb_type=progress_cb_type,
+            enable_fallback=enable_fallback,
+        )
+        return run_coroutine(coroutine)

supervisely-6.73.324/supervisely/app/fastapi/utils.py

@@ -0,0 +1,28 @@
+import asyncio
+import concurrent.futures
+
+
+def run_sync(coroutine):
+    """
+    Runs an asynchronous coroutine in a separate thread and waits for its result.
+    It is useful for running async functions in a synchronous
+    environment.
+
+    This method creates a new thread using ThreadPoolExecutor and executes the coroutine
+    inside a new event loop.
+
+    ⚠️ Note: This function creates a new event loop every time it is called,
+    which can cause issues when using objects tied to a specific loop (e.g., asyncio.Semaphore).
+
+    :param coroutine: coroutine to run
+    :return: result of coroutine
+    """
+    try:
+        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+            result = executor.submit(
+                lambda coroutine_to_exec: asyncio.run(coroutine_to_exec), coroutine
+            ).result()
+        return result
+    except RuntimeError as ex:
+        print(repr(ex))
+        return None

{supervisely-6.73.323 → supervisely-6.73.324}/supervisely/nn/training/train_app.py

@@ -39,7 +39,7 @@ from supervisely import (
     is_production,
     logger,
 )
-from supervisely._utils import abs_url, get_filename_from_headers, sync_call
+from supervisely._utils import abs_url, get_filename_from_headers
 from supervisely.api.file_api import FileInfo
 from supervisely.app import get_synced_data_dir
 from supervisely.app.widgets import Progress
@@ -60,7 +60,7 @@ from supervisely.nn.utils import ModelSource
 from supervisely.output import set_directory
 from supervisely.project.download import (
     copy_from_cache,
-    download_async_or_sync,
+    download_fast,
     download_to_cache,
     get_cache_size,
     is_cached,
@@ -806,7 +806,7 @@ class TrainApp:
         with self.progress_bar_main(message="Downloading input data", total=total_images) as pbar:
             logger.debug("Downloading project data without cache")
             self.progress_bar_main.show()
-            download_async_or_sync(
+            download_fast(
                 api=self._api,
                 project_id=self.project_id,
                 dest_dir=self.project_dir,
@@ -1619,14 +1619,13 @@ class TrainApp:
             unit_scale=True,
         ) as upload_artifacts_pbar:
             self.progress_bar_main.show()
-            remote_dir = sync_call(
-                self._api.file.upload_directory_async(
-                    team_id=self.team_id,
-                    local_dir=local_demo_dir,
-                    remote_dir=remote_demo_dir,
-                    progress_size_cb=upload_artifacts_pbar.update,
-                )
+            remote_dir = self._api.file.upload_directory_fast(
+                team_id=self.team_id,
+                local_dir=local_demo_dir,
+                remote_dir=remote_demo_dir,
+                progress_cb=upload_artifacts_pbar.update,
             )
+
             self.progress_bar_main.hide()
 
     def _get_train_val_splits_for_app_state(self) -> Dict:
@@ -1733,13 +1732,11 @@ class TrainApp:
             unit_scale=True,
         ) as upload_artifacts_pbar:
             self.progress_bar_main.show()
-            remote_dir = sync_call(
-                self._api.file.upload_directory_async(
-                    team_id=self.team_id,
-                    local_dir=self.output_dir,
-                    remote_dir=remote_artifacts_dir,
-                    progress_size_cb=upload_artifacts_pbar.update,
-                )
+            remote_dir = self._api.file.upload_directory_fast(
+                team_id=self.team_id,
+                local_dir=self.output_dir,
+                remote_dir=remote_artifacts_dir,
+                progress_cb=upload_artifacts_pbar.update,
             )
             self.progress_bar_main.hide()
 
@@ -2524,13 +2521,11 @@ class TrainApp:
             logger.debug(f"Uploading {len(export_weights)} export weights of size {size} bytes")
             logger.debug(f"Destination paths: {file_dest_paths}")
             self.progress_bar_main.show()
-            sync_call(
-                self._api.file.upload_bulk_async(
-                    team_id=self.team_id,
-                    src_paths=export_weights.values(),
-                    dst_paths=file_dest_paths,
-                    progress_cb=export_upload_main_pbar.update,
-                )
+            self._api.file.upload_bulk_fast(
+                team_id=self.team_id,
+                src_paths=export_weights.values(),
+                dst_paths=file_dest_paths,
+                progress_cb=export_upload_main_pbar.update,
             )
 
         self.progress_bar_main.hide()

{supervisely-6.73.323 → supervisely-6.73.324}/supervisely/project/download.py

@@ -281,6 +281,34 @@ def download_async_or_sync(
         )
 
 
+def download_fast(
+    api: Api,
+    project_id: int,
+    dest_dir: str,
+    dataset_ids: Optional[List[int]] = None,
+    log_progress: bool = True,
+    progress_cb: Optional[Union[tqdm, Callable]] = None,
+    semaphore: Optional[asyncio.Semaphore] = None,
+    **kwargs,
+) -> None:
+    """
+    Download project in a fast mode.
+    Items are downloaded asynchronously. If an error occurs, the method will fallback to synchronous download.
+    Automatically detects project type.
+    You can pass :class:`ProjectInfo` as `project_info` kwarg to avoid additional API requests.
+    """
+    download_async_or_sync(
+        api=api,
+        project_id=project_id,
+        dest_dir=dest_dir,
+        dataset_ids=dataset_ids,
+        log_progress=log_progress,
+        progress_cb=progress_cb,
+        semaphore=semaphore,
+        **kwargs,
+    )
+
+
 def _get_cache_dir(project_id: int, dataset_path: str = None) -> str:
     p = os.path.join(apps_cache_dir(), str(project_id))
     if dataset_path is not None:
@@ -468,7 +496,7 @@ def _download_project_to_cache(
     if len(dataset_infos) == 0:
         logger.debug("No datasets to download")
         return
-    download_async_or_sync(
+    download_fast(
         api=api,
         project_id=project_id,
         dest_dir=cached_project_dir,

{supervisely-6.73.323 → supervisely-6.73.324/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.323
+Version: 6.73.324
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

supervisely-6.73.324/tests/test_async_file_api.py

@@ -0,0 +1,170 @@
+import asyncio
+import os
+import time
+
+from tqdm import tqdm
+
+from supervisely import Api, logger
+from supervisely._utils import run_coroutine
+from supervisely.io.fs import clean_dir, ensure_base_path
+
+# LOG_LEVEL = "INFO"
+LOG_LEVEL = "DEBUG"
+api = Api.from_env()
+
+api.logger.setLevel(LOG_LEVEL)
+
+TEAM_ID = 10
+user_path = os.path.expanduser("~")
+source_path = f"{user_path}/test_project_download/"
+save_path = f"{user_path}/Work/test_project_download/"
+logger.info(f"Save path: {save_path}")
+remote_path = "/videos/"
+remote_upload_test_sync = "/test_sync/"
+remote_upload_test_async = "/test_async/"
+ensure_base_path(save_path)
+clean_dir(save_path)
+files = (
+    ("12391768_3840_2160_30fps.mp4", f"{remote_path}12391768_3840_2160_30fps.mp4"),
+    ("MOV_h264.mov", f"{remote_path}MOV_h264.mov"),
+    ("MP4_HEVC.mp4", f"{remote_path}MP4_HEVC.mp4"),
+)
+file_name = "vids.tar"
+
+
+async def download_files():
+    sema = asyncio.Semaphore(10)
+    tasks = []
+    for name, path in files:
+        progress = tqdm(total=None, desc=f"Downloading {name}", unit="B", unit_scale=True)
+        task = api.file.download_async(TEAM_ID, path, save_path + name, sema, progress_cb=progress)
+        tasks.append(task)
+    await asyncio.gather(*tasks)
+
+
+def main_db():
+    remote_paths = [path for _, path in files]
+    names = [name for name, _ in files]
+    sizeb_list = [api.file.get_info_by_path(TEAM_ID, path).sizeb for _, path in files]
+    sizeb = sum(sizeb_list)
+    save_paths = [save_path + name for name in names]
+    progress = tqdm(total=sizeb, desc="Downloading files", unit="B", unit_scale=True)
+    download_coro = api.file.download_bulk_async(
+        TEAM_ID, remote_paths, save_paths, progress_cb=progress
+    )
+    run_coroutine(download_coro)
+
+
+def maind_df():
+    run_coroutine(download_files())
+
+
+def main_ip():
+    start = time.time()
+    # os.environ["CONTEXT_SLYFILE"] = "/test/test_input.tar"
+    os.environ["FOLDER"] = "/videos/"
+    os.environ["TEAM_ID"] = str(TEAM_ID)
+    download_coro = api.file.download_input_async(save_path, show_progress=True)
+    run_coroutine(download_coro)
+    end = time.time()
+    print(f"Time taken for download input async: {end-start}")
+
+
+def main_dd():
+    start = time.time()
+    download_coro = api.file.download_directory_async(
+        TEAM_ID, remote_path, save_path, semaphore=asyncio.Semaphore(100)
+    )
+    run_coroutine(download_coro)
+    end = time.time()
+    print(f"Time taken for download dir async: {end-start}")
+
+
+def compare_dir_download():
+    main_dd()
+
+    clean_dir(save_path)
+
+    start = time.time()
+    api.file.download_directory(TEAM_ID, remote_path, save_path)
+    end = time.time()
+    print(f"Time taken for download dir: {end-start}")
+
+
+def upload():
+    total = os.path.getsize(source_path + file_name)
+    process_cb = tqdm(total=total, desc="Uploading files", unit="B", unit_scale=True)
+    upload_coro = api.file.upload_async(
+        TEAM_ID,
+        source_path + file_name,
+        remote_path + file_name,
+        progress_cb=process_cb,
+    )
+    start = time.monotonic()
+    run_coroutine(upload_coro)
+    end = time.monotonic()
+    logger.info(f"Time taken for upload async: {(end-start) / 60}")
+
+
+def upload_bulk():
+    total = 0
+    src_paths = []
+    dst_paths = []
+    for file in os.listdir(source_path):
+        path = os.path.join(source_path, file)
+        src_paths.append(path)
+        dst_paths.append(remote_path + file)
+        total += os.path.getsize(path)
+    process_cb = tqdm(total=total, desc="Uploading files", unit="B", unit_scale=True)
+    upload_coro = api.file.upload_bulk_async(
+        TEAM_ID,
+        src_paths,
+        dst_paths,
+        progress_cb=process_cb,
+    )
+    start = time.monotonic()
+    run_coroutine(upload_coro)
+    end = time.monotonic()
+    logger.info(f"Time taken for upload bulk async: {(end-start) / 60}")
+
+
+def main_ud():
+    total = 0
+    for file in os.listdir(source_path):
+        path = os.path.join(source_path, file)
+        total += os.path.getsize(path)
+
+    process_cb = tqdm(total=total, desc="Uploading files", unit="B", unit_scale=True)
+    start = time.monotonic()
+    api.file.upload_directory(
+        TEAM_ID,
+        source_path,
+        remote_upload_test_sync,
+        progress_size_cb=process_cb,
+    )
+    end = time.monotonic()
+    logger.info(f"Time taken for upload dir: {end-start}")
+
+    process_cb_as = tqdm(total=total, desc="Uploading files async", unit="B", unit_scale=True)
+    coro = api.file.upload_directory_async(
+        TEAM_ID,
+        source_path,
+        remote_upload_test_async,
+        progress_size_cb=process_cb_as,
+    )
+    start = time.monotonic()
+    run_coroutine(coro)
+    end = time.monotonic()
+    logger.info(f"Time taken for upload dir async: {end-start}")
+
+
+if __name__ == "__main__":
+    try:
+        # maind_df()  # to download and save files
+        # main_dd()  # to download and save files as folder
+        # main_db()  # to download and save files in bulk
+        # main_ip()  # to download input file
+        # compare_dir_download()  # to compare download time between async and sync
+        main_ud()
+    except KeyboardInterrupt:
+        logger.info("Stopped by user")