supervisely 6.73.320__tar.gz → 6.73.321__tar.gz

{supervisely-6.73.320/supervisely.egg-info → supervisely-6.73.321}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.320
+Version: 6.73.321
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.320 → supervisely-6.73.321}/supervisely/_utils.py

@@ -471,6 +471,39 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
             return loop
 
 
+def sync_call(coro):
+    """
+    This function is used to run asynchronous functions in synchronous context.
+
+    :param coro: Asynchronous function.
+    :type coro: Coroutine
+    :return: Result of the asynchronous function.
+    :rtype: Any
+
+    :Usage example:
+
+    .. code-block:: python
+
+            from supervisely.utils import sync_call
+
+            async def async_function():
+                await asyncio.sleep(1)
+                return "Hello, World!"
+            coro = async_function()
+            result = sync_call(coro)
+            print(result)
+            # Output: Hello, World!
+    """
+
+    loop = get_or_create_event_loop()
+
+    if loop.is_running():
+        future = asyncio.run_coroutine_threadsafe(coro, loop=loop)
+        return future.result()
+    else:
+        return loop.run_until_complete(coro)
+
+
 def get_filename_from_headers(url):
     try:
         response = requests.head(url, allow_redirects=True)

{supervisely-6.73.320 → supervisely-6.73.321}/supervisely/api/api.py

@@ -1443,6 +1443,7 @@ class Api:
         chunk_size: int = 8192,
         use_public_api: Optional[bool] = True,
         timeout: httpx._types.TimeoutTypes = 60,
+        **kwargs,
     ) -> AsyncGenerator:
         """
         Performs asynchronous streaming GET or POST request to server with given parameters.
@@ -1486,18 +1487,19 @@ class Api:
         else:
             headers = {**self.headers, **headers}
 
-        if isinstance(data, (bytes, Generator)):
-            content = data
-            json_body = None
-            params = None
-        elif isinstance(data, Dict):
-            json_body = {**data, **self.additional_fields}
-            content = None
-            params = None
+        params = kwargs.get("params", None)
+        if "content" in kwargs or "json_body" in kwargs:
+            content = kwargs.get("content", None)
+            json_body = kwargs.get("json_body", None)
         else:
-            params = data
-            content = None
-            json_body = None
+            if isinstance(data, (bytes, Generator)):
+                content = data
+                json_body = None
+            elif isinstance(data, Dict):
+                json_body = {**data, **self.additional_fields}
+                content = None
+            else:
+                raise ValueError("Data should be either bytes or dict")
 
         if range_start is not None or range_end is not None:
             headers["Range"] = f"bytes={range_start or ''}-{range_end or ''}"
@@ -1512,17 +1514,19 @@ class Api:
                         url,
                         content=content,
                         json=json_body,
-                        params=params,
                         headers=headers,
                         timeout=timeout,
+                        params=params,
                     )
                 elif method_type == "GET":
                     response = self.async_httpx_client.stream(
                         method_type,
                         url,
-                        json=json_body or params,
+                        content=content,
+                        json=json_body,
                         headers=headers,
                         timeout=timeout,
+                        params=params,
                     )
                 else:
                     raise NotImplementedError(

{supervisely-6.73.320 → supervisely-6.73.321}/supervisely/api/file_api.py

@@ -33,7 +33,9 @@ from supervisely.io.fs import (
     get_file_name,
     get_file_name_with_ext,
     get_file_size,
+    get_or_create_event_loop,
     list_files_recursively,
+    list_files_recursively_async,
     silent_remove,
 )
 from supervisely.io.fs_cache import FileCache
@@ -2041,7 +2043,7 @@ class FileApi(ModuleApiBase):
         # check_hash: bool = True, #TODO add with resumaple api
         progress_cb: Optional[Union[tqdm, Callable]] = None,
         progress_cb_type: Literal["number", "size"] = "size",
-    ) -> httpx.Response:
+    ) -> None:
         """
         Upload file from local path to Team Files asynchronously.
 
@@ -2057,8 +2059,8 @@ class FileApi(ModuleApiBase):
         :type progress_cb: tqdm or callable, optional
         :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
         :type progress_cb_type: Literal["number", "size"], optional
-        :return: Response from API.
-        :rtype: :class:`httpx.Response`
+        :return: None
+        :rtype: :class:`NoneType`
         :Usage example:
 
             .. code-block:: python
@@ -2087,17 +2089,30 @@ class FileApi(ModuleApiBase):
         }
         if semaphore is None:
             semaphore = self._api.get_default_semaphore()
+        logger.debug(f"Uploading with async to: {dst}. Semaphore: {semaphore}")
         async with semaphore:
             async with aiofiles.open(src, "rb") as fd:
-                item = await fd.read()
-                response = await self._api.post_async(
-                    api_method, content=item, params=json_body, headers=headers
-                )
-                if progress_cb is not None and progress_cb_type == "size":
-                    progress_cb(len(item))
+
+                async def file_chunk_generator():
+                    while True:
+                        chunk = await fd.read(8 * 1024 * 1024)
+                        if not chunk:
+                            break
+                        if progress_cb is not None and progress_cb_type == "size":
+                            progress_cb(len(chunk))
+                        yield chunk
+
+                async for chunk, _ in self._api.stream_async(
+                    method=api_method,
+                    method_type="POST",
+                    data=file_chunk_generator(),  # added as required, but not used inside
+                    headers=headers,
+                    content=file_chunk_generator(),  # used instead of data inside stream_async
+                    params=json_body,
+                ):
+                    pass
                 if progress_cb is not None and progress_cb_type == "number":
                     progress_cb(1)
-                return response
 
     async def upload_bulk_async(
         self,
@@ -2109,6 +2124,7 @@ class FileApi(ModuleApiBase):
         # check_hash: bool = True, #TODO add with resumaple api
         progress_cb: Optional[Union[tqdm, Callable]] = None,
         progress_cb_type: Literal["number", "size"] = "size",
+        enable_fallback: Optional[bool] = True,
     ) -> None:
         """
         Upload multiple files from local paths to Team Files asynchronously.
@@ -2125,6 +2141,8 @@ class FileApi(ModuleApiBase):
         :type progress_cb: tqdm or callable, optional
         :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
         :type progress_cb_type: Literal["number", "size"], optional
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
         :return: None
         :rtype: :class:`NoneType`
         :Usage example:
@@ -2153,19 +2171,134 @@ class FileApi(ModuleApiBase):
                     api.file.upload_bulk_async(8, paths_to_files, paths_to_save)
                 )
         """
-        if semaphore is None:
-            semaphore = self._api.get_default_semaphore()
-        tasks = []
-        for s, d in zip(src_paths, dst_paths):
-            task = self.upload_async(
-                team_id,
-                s,
-                d,
-                semaphore=semaphore,
-                # chunk_size=chunk_size, #TODO add with resumaple api
-                # check_hash=check_hash, #TODO add with resumaple api
-                progress_cb=progress_cb,
-                progress_cb_type=progress_cb_type,
+        try:
+            if semaphore is None:
+                semaphore = self._api.get_default_semaphore()
+            tasks = []
+            for src, dst in zip(src_paths, dst_paths):
+                task = asyncio.create_task(
+                    self.upload_async(
+                        team_id=team_id,
+                        src=src,
+                        dst=dst,
+                        semaphore=semaphore,
+                        # chunk_size=chunk_size, #TODO add with resumaple api
+                        # check_hash=check_hash, #TODO add with resumaple api
+                        progress_cb=progress_cb,
+                        progress_cb_type=progress_cb_type,
+                    )
+                )
+                tasks.append(task)
+            for task in tasks:
+                await task
+        except Exception as e:
+            if enable_fallback:
+                logger.warning(
+                    f"Upload files bulk asynchronously failed. Fallback to synchronous upload.",
+                    exc_info=True,
+                )
+                if progress_cb is not None and progress_cb_type == "number":
+                    logger.warning(
+                        "Progress callback type 'number' is not supported for synchronous upload. "
+                        "Progress callback will be disabled."
+                    )
+                    progress_cb = None
+                self.upload_bulk(
+                    team_id=team_id,
+                    src_paths=src_paths,
+                    dst_paths=dst_paths,
+                    progress_cb=progress_cb,
+                )
+            else:
+                raise e
+
+    async def upload_directory_async(
+        self,
+        team_id: int,
+        local_dir: str,
+        remote_dir: str,
+        change_name_if_conflict: Optional[bool] = True,
+        progress_size_cb: Optional[Union[tqdm, Callable]] = None,
+        replace_if_conflict: Optional[bool] = False,
+        enable_fallback: Optional[bool] = True,
+    ) -> str:
+        """
+        Upload Directory to Team Files from local path.
+        Files are uploaded asynchronously.
+
+        :param team_id: Team ID in Supervisely.
+        :type team_id: int
+        :param local_dir: Path to local Directory.
+        :type local_dir: str
+        :param remote_dir: Path to Directory in Team Files.
+        :type remote_dir: str
+        :param change_name_if_conflict: Checks if given name already exists and adds suffix to the end of the name.
+        :type change_name_if_conflict: bool, optional
+        :param progress_size_cb: Function for tracking download progress.
+        :type progress_size_cb: Progress, optional
+        :param replace_if_conflict: If True, replace existing dir.
+        :type replace_if_conflict: bool, optional
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
+        :return: Path to Directory in Team Files
+        :rtype: :class:`str`
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            path_to_dir = "/My_App_Test/ds1"
+            local_path = "/home/admin/Downloads/My_local_test"
+
+            api.file.upload_directory(9, local_path, path_to_dir)
+        """
+        try:
+            if not remote_dir.startswith("/"):
+                remote_dir = "/" + remote_dir
+
+            if self.dir_exists(team_id, remote_dir):
+                if change_name_if_conflict is True:
+                    res_remote_dir = self.get_free_dir_name(team_id, remote_dir)
+                elif replace_if_conflict is True:
+                    res_remote_dir = remote_dir
+                else:
+                    raise FileExistsError(
+                        f"Directory {remote_dir} already exists in your team (id={team_id})"
+                    )
+            else:
+                res_remote_dir = remote_dir
+
+            local_files = await list_files_recursively_async(local_dir)
+            dir_prefix = local_dir.rstrip("/") + "/"
+            remote_files = [
+                res_remote_dir.rstrip("/") + "/" + file[len(dir_prefix) :] for file in local_files
+            ]
+
+            await self.upload_bulk_async(
+                team_id=team_id,
+                src_paths=local_files,
+                dst_paths=remote_files,
+                progress_cb=progress_size_cb,
             )
-            tasks.append(task)
-        await asyncio.gather(*tasks)
+        except Exception as e:
+            if enable_fallback:
+                logger.warning(
+                    f"Upload directory asynchronously failed. Fallback to synchronous upload.",
+                    exc_info=True,
+                )
+                res_remote_dir = self.upload_directory(
+                    team_id=team_id,
+                    local_dir=local_dir,
+                    remote_dir=res_remote_dir,
+                    change_name_if_conflict=change_name_if_conflict,
+                    progress_size_cb=progress_size_cb,
+                    replace_if_conflict=replace_if_conflict,
+                )
+            else:
+                raise e
+        return res_remote_dir

{supervisely-6.73.320 → supervisely-6.73.321}/supervisely/io/fs.py

@@ -205,15 +205,19 @@ def list_files_recursively(
             for filename in file_names:
                 yield os.path.join(dir_name, filename)
 
-    valid_extensions = valid_extensions if ignore_valid_extensions_case is False else [ext.lower() for ext in valid_extensions]
+    valid_extensions = (
+        valid_extensions
+        if ignore_valid_extensions_case is False
+        else [ext.lower() for ext in valid_extensions]
+    )
     files = []
     for file_path in file_path_generator():
         file_ext = get_file_ext(file_path)
         if ignore_valid_extensions_case:
             file_ext.lower()
-        if (
-            valid_extensions is None or file_ext in valid_extensions
-        ) and (filter_fn is None or filter_fn(file_path)):
+        if (valid_extensions is None or file_ext in valid_extensions) and (
+            filter_fn is None or filter_fn(file_path)
+        ):
             files.append(file_path)
     return files
 
@@ -1558,3 +1562,76 @@ async def touch_async(path: str) -> None:
     async with aiofiles.open(path, "a"):
         loop = get_or_create_event_loop()
         await loop.run_in_executor(None, os.utime, path, None)
+
+
+async def list_files_recursively_async(
+    dir_path: str,
+    valid_extensions: Optional[List[str]] = None,
+    filter_fn: Optional[Callable[[str], bool]] = None,
+    ignore_valid_extensions_case: bool = False,
+) -> List[str]:
+    """
+    Recursively list files in the directory asynchronously.
+    Returns list with all file paths.
+    Can be filtered by valid extensions and filter function.
+
+    :param dir_path: Target directory path.
+    :type dir_path: str
+    :param valid_extensions: List of valid extensions. Default is None.
+    :type valid_extensions: Optional[List[str]]
+    :param filter_fn: Filter function. Default is None.
+    :type filter_fn: Optional[Callable[[str], bool]]
+    :param ignore_valid_extensions_case: Ignore case when checking valid extensions. Default is False.
+    :type ignore_valid_extensions_case: bool
+    :returns: List of file paths
+    :rtype: List[str]
+
+    :Usage example:
+    
+         .. code-block:: python
+    
+            import supervisely as sly
+    
+            dir_path = '/home/admin/work/projects/examples'
+            loop = sly.utils.get_or_create_event_loop()
+            coro = sly.fs.list_files_recursively_async(dir_path)
+            if loop.is_running():
+                future = asyncio.run_coroutine_threadsafe(coro, loop)
+                files = future.result()
+            else:
+                files = loop.run_until_complete(coro)
+    """
+
+    def sync_file_list():
+        if valid_extensions and ignore_valid_extensions_case:
+            valid_extensions_set = set(map(str.lower, valid_extensions))
+        else:
+            valid_extensions_set = set(valid_extensions) if valid_extensions else None
+
+        files = []
+        for dir_name, _, file_names in os.walk(dir_path):
+            full_paths = [os.path.join(dir_name, filename) for filename in file_names]
+
+            if valid_extensions_set:
+                full_paths = [
+                    fp
+                    for fp in full_paths
+                    if (
+                        ext := (
+                            os.path.splitext(fp)[1].lower()
+                            if ignore_valid_extensions_case
+                            else os.path.splitext(fp)[1]
+                        )
+                    )
+                    in valid_extensions_set
+                ]
+
+            if filter_fn:
+                full_paths = [fp for fp in full_paths if filter_fn(fp)]
+
+            files.extend(full_paths)
+
+        return files
+
+    loop = get_or_create_event_loop()
+    return await loop.run_in_executor(None, sync_file_list)

{supervisely-6.73.320 → supervisely-6.73.321}/supervisely/nn/training/train_app.py

@@ -39,7 +39,7 @@ from supervisely import (
     is_production,
     logger,
 )
-from supervisely._utils import abs_url, get_filename_from_headers
+from supervisely._utils import abs_url, get_filename_from_headers, sync_call
 from supervisely.api.file_api import FileInfo
 from supervisely.app import get_synced_data_dir
 from supervisely.app.widgets import Progress
@@ -60,6 +60,7 @@ from supervisely.nn.utils import ModelSource
 from supervisely.output import set_directory
 from supervisely.project.download import (
     copy_from_cache,
+    download_async_or_sync,
     download_to_cache,
     get_cache_size,
     is_cached,
@@ -803,8 +804,9 @@ class TrainApp:
         :type total_images: int
         """
         with self.progress_bar_main(message="Downloading input data", total=total_images) as pbar:
+            logger.debug("Downloading project data without cache")
             self.progress_bar_main.show()
-            download_project(
+            download_async_or_sync(
                 api=self._api,
                 project_id=self.project_id,
                 dest_dir=self.project_dir,
@@ -834,6 +836,7 @@ class TrainApp:
 
         logger.info(self._get_cache_log_message(cached, to_download))
         with self.progress_bar_main(message="Downloading input data", total=total_images) as pbar:
+            logger.debug("Downloading project data with cache")
             self.progress_bar_main.show()
             download_to_cache(
                 api=self._api,
@@ -1603,10 +1606,12 @@ class TrainApp:
             logger.info(f"Demo directory '{local_demo_dir}' does not exist")
             return
 
-        logger.debug(f"Uploading demo files to Supervisely")
         remote_demo_dir = join(remote_dir, "demo")
         local_files = sly_fs.list_files_recursively(local_demo_dir)
         total_size = sum([sly_fs.get_file_size(file_path) for file_path in local_files])
+        logger.debug(
+            f"Uploading demo files of total size {total_size} bytes to Supervisely Team Files directory '{remote_demo_dir}'"
+        )
         with self.progress_bar_main(
             message="Uploading demo files to Team Files",
             total=total_size,
@@ -1614,11 +1619,13 @@ class TrainApp:
             unit_scale=True,
         ) as upload_artifacts_pbar:
             self.progress_bar_main.show()
-            remote_dir = self._api.file.upload_directory(
-                self.team_id,
-                local_demo_dir,
-                remote_demo_dir,
-                progress_size_cb=upload_artifacts_pbar,
+            remote_dir = sync_call(
+                self._api.file.upload_directory_async(
+                    team_id=self.team_id,
+                    local_dir=local_demo_dir,
+                    remote_dir=remote_demo_dir,
+                    progress_size_cb=upload_artifacts_pbar.update,
+                )
             )
             self.progress_bar_main.hide()
 
@@ -1690,11 +1697,12 @@ class TrainApp:
         Path: /experiments/{project_id}_{project_name}/{task_id}_{framework_name}/
         Example path: /experiments/43192_Apples/68271_rt-detr/
         """
-        logger.info(f"Uploading directory: '{self.output_dir}' to Supervisely")
         task_id = self.task_id
 
         remote_artifacts_dir = f"/{self._experiments_dir_name}/{self.project_id}_{self.project_name}/{task_id}_{self.framework_name}/"
-
+        logger.info(
+            f"Uploading artifacts directory: '{self.output_dir}' to Supervisely Team Files directory '{remote_artifacts_dir}'"
+        )
         # Clean debug directory if exists
         if task_id == "debug-session":
             if self._api.file.dir_exists(self.team_id, f"{remote_artifacts_dir}/", True):
@@ -1725,11 +1733,13 @@ class TrainApp:
             unit_scale=True,
         ) as upload_artifacts_pbar:
             self.progress_bar_main.show()
-            remote_dir = self._api.file.upload_directory(
-                self.team_id,
-                self.output_dir,
-                remote_artifacts_dir,
-                progress_size_cb=upload_artifacts_pbar,
+            remote_dir = sync_call(
+                self._api.file.upload_directory_async(
+                    team_id=self.team_id,
+                    local_dir=self.output_dir,
+                    remote_dir=remote_artifacts_dir,
+                    progress_size_cb=upload_artifacts_pbar.update,
+                )
             )
             self.progress_bar_main.hide()
 
@@ -2088,6 +2098,7 @@ class TrainApp:
                 self.project_info.id, version_id=project_version_id
             )
 
+            file_info = None
             if self.model_source == ModelSource.CUSTOM:
                 file_info = self._api.file.get_info_by_path(
                     self.team_id,
@@ -2496,32 +2507,33 @@ class TrainApp:
     def _upload_export_weights(
         self, export_weights: Dict[str, str], remote_dir: str
     ) -> Dict[str, str]:
+        """Uploads export weights (any other specified formats) to Supervisely Team Files.
+        The default export is handled by the `_upload_artifacts` method."""
+        file_dest_paths = []
+        size = 0
+        for path in export_weights.values():
+            file_name = sly_fs.get_file_name_with_ext(path)
+            file_dest_paths.append(join(remote_dir, self._export_dir_name, file_name))
+            size += sly_fs.get_file_size(path)
         with self.progress_bar_main(
-            message="Uploading export weights",
-            total=len(export_weights),
+            message=f"Uploading {len(export_weights)} export weights",
+            total=size,
+            unit="B",
+            unit_scale=True,
         ) as export_upload_main_pbar:
+            logger.debug(f"Uploading {len(export_weights)} export weights of size {size} bytes")
+            logger.debug(f"Destination paths: {file_dest_paths}")
             self.progress_bar_main.show()
-            for path in export_weights.values():
-                file_name = sly_fs.get_file_name_with_ext(path)
-                file_size = sly_fs.get_file_size(path)
-                with self.progress_bar_secondary(
-                    message=f"Uploading '{file_name}' ",
-                    total=file_size,
-                    unit="bytes",
-                    unit_scale=True,
-                ) as export_upload_secondary_pbar:
-                    self.progress_bar_secondary.show()
-                    destination_path = join(remote_dir, self._export_dir_name, file_name)
-                    self._api.file.upload(
-                        self.team_id,
-                        path,
-                        destination_path,
-                        export_upload_secondary_pbar,
-                    )
-                export_upload_main_pbar.update(1)
+            sync_call(
+                self._api.file.upload_bulk_async(
+                    team_id=self.team_id,
+                    src_paths=export_weights.values(),
+                    dst_paths=file_dest_paths,
+                    progress_cb=export_upload_main_pbar.update,
+                )
+            )
 
         self.progress_bar_main.hide()
-        self.progress_bar_secondary.hide()
 
         remote_export_weights = {
             runtime: join(self._export_dir_name, sly_fs.get_file_name_with_ext(path))

{supervisely-6.73.320 → supervisely-6.73.321/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.320
+Version: 6.73.321
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely