supervisely 6.73.319__tar.gz → 6.73.321__tar.gz

{supervisely-6.73.319/supervisely.egg-info → supervisely-6.73.321}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.319
+Version: 6.73.321
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.319 → supervisely-6.73.321}/supervisely/_utils.py

@@ -471,6 +471,39 @@ def get_or_create_event_loop() -> asyncio.AbstractEventLoop:
             return loop
 
 
+def sync_call(coro):
+    """
+    This function is used to run asynchronous functions in synchronous context.
+
+    :param coro: Asynchronous function.
+    :type coro: Coroutine
+    :return: Result of the asynchronous function.
+    :rtype: Any
+
+    :Usage example:
+
+    .. code-block:: python
+
+            from supervisely.utils import sync_call
+
+            async def async_function():
+                await asyncio.sleep(1)
+                return "Hello, World!"
+            coro = async_function()
+            result = sync_call(coro)
+            print(result)
+            # Output: Hello, World!
+    """
+
+    loop = get_or_create_event_loop()
+
+    if loop.is_running():
+        future = asyncio.run_coroutine_threadsafe(coro, loop=loop)
+        return future.result()
+    else:
+        return loop.run_until_complete(coro)
+
+
 def get_filename_from_headers(url):
     try:
         response = requests.head(url, allow_redirects=True)

{supervisely-6.73.319 → supervisely-6.73.321}/supervisely/api/api.py

@@ -1443,6 +1443,7 @@ class Api:
         chunk_size: int = 8192,
         use_public_api: Optional[bool] = True,
         timeout: httpx._types.TimeoutTypes = 60,
+        **kwargs,
     ) -> AsyncGenerator:
         """
         Performs asynchronous streaming GET or POST request to server with given parameters.
@@ -1486,18 +1487,19 @@ class Api:
         else:
             headers = {**self.headers, **headers}
 
-        if isinstance(data, (bytes, Generator)):
-            content = data
-            json_body = None
-            params = None
-        elif isinstance(data, Dict):
-            json_body = {**data, **self.additional_fields}
-            content = None
-            params = None
+        params = kwargs.get("params", None)
+        if "content" in kwargs or "json_body" in kwargs:
+            content = kwargs.get("content", None)
+            json_body = kwargs.get("json_body", None)
         else:
-            params = data
-            content = None
-            json_body = None
+            if isinstance(data, (bytes, Generator)):
+                content = data
+                json_body = None
+            elif isinstance(data, Dict):
+                json_body = {**data, **self.additional_fields}
+                content = None
+            else:
+                raise ValueError("Data should be either bytes or dict")
 
         if range_start is not None or range_end is not None:
             headers["Range"] = f"bytes={range_start or ''}-{range_end or ''}"
@@ -1512,17 +1514,19 @@ class Api:
                         url,
                         content=content,
                         json=json_body,
-                        params=params,
                         headers=headers,
                         timeout=timeout,
+                        params=params,
                     )
                 elif method_type == "GET":
                     response = self.async_httpx_client.stream(
                         method_type,
                         url,
-                        json=json_body or params,
+                        content=content,
+                        json=json_body,
                         headers=headers,
                         timeout=timeout,
+                        params=params,
                     )
                 else:
                     raise NotImplementedError(

{supervisely-6.73.319 → supervisely-6.73.321}/supervisely/api/file_api.py

@@ -33,7 +33,9 @@ from supervisely.io.fs import (
     get_file_name,
     get_file_name_with_ext,
     get_file_size,
+    get_or_create_event_loop,
     list_files_recursively,
+    list_files_recursively_async,
     silent_remove,
 )
 from supervisely.io.fs_cache import FileCache
@@ -2041,7 +2043,7 @@ class FileApi(ModuleApiBase):
         # check_hash: bool = True, #TODO add with resumaple api
         progress_cb: Optional[Union[tqdm, Callable]] = None,
         progress_cb_type: Literal["number", "size"] = "size",
-    ) -> httpx.Response:
+    ) -> None:
         """
         Upload file from local path to Team Files asynchronously.
 
@@ -2057,8 +2059,8 @@ class FileApi(ModuleApiBase):
         :type progress_cb: tqdm or callable, optional
         :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
         :type progress_cb_type: Literal["number", "size"], optional
-        :return: Response from API.
-        :rtype: :class:`httpx.Response`
+        :return: None
+        :rtype: :class:`NoneType`
         :Usage example:
 
             .. code-block:: python
@@ -2087,17 +2089,30 @@ class FileApi(ModuleApiBase):
         }
         if semaphore is None:
             semaphore = self._api.get_default_semaphore()
+        logger.debug(f"Uploading with async to: {dst}. Semaphore: {semaphore}")
         async with semaphore:
             async with aiofiles.open(src, "rb") as fd:
-                item = await fd.read()
-                response = await self._api.post_async(
-                    api_method, content=item, params=json_body, headers=headers
-                )
-                if progress_cb is not None and progress_cb_type == "size":
-                    progress_cb(len(item))
+
+                async def file_chunk_generator():
+                    while True:
+                        chunk = await fd.read(8 * 1024 * 1024)
+                        if not chunk:
+                            break
+                        if progress_cb is not None and progress_cb_type == "size":
+                            progress_cb(len(chunk))
+                        yield chunk
+
+                async for chunk, _ in self._api.stream_async(
+                    method=api_method,
+                    method_type="POST",
+                    data=file_chunk_generator(),  # added as required, but not used inside
+                    headers=headers,
+                    content=file_chunk_generator(),  # used instead of data inside stream_async
+                    params=json_body,
+                ):
+                    pass
                 if progress_cb is not None and progress_cb_type == "number":
                     progress_cb(1)
-                return response
 
     async def upload_bulk_async(
         self,
@@ -2109,6 +2124,7 @@ class FileApi(ModuleApiBase):
         # check_hash: bool = True, #TODO add with resumaple api
         progress_cb: Optional[Union[tqdm, Callable]] = None,
         progress_cb_type: Literal["number", "size"] = "size",
+        enable_fallback: Optional[bool] = True,
     ) -> None:
         """
         Upload multiple files from local paths to Team Files asynchronously.
@@ -2125,6 +2141,8 @@ class FileApi(ModuleApiBase):
         :type progress_cb: tqdm or callable, optional
         :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
         :type progress_cb_type: Literal["number", "size"], optional
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
         :return: None
         :rtype: :class:`NoneType`
         :Usage example:
@@ -2153,19 +2171,134 @@ class FileApi(ModuleApiBase):
                     api.file.upload_bulk_async(8, paths_to_files, paths_to_save)
                 )
         """
-        if semaphore is None:
-            semaphore = self._api.get_default_semaphore()
-        tasks = []
-        for s, d in zip(src_paths, dst_paths):
-            task = self.upload_async(
-                team_id,
-                s,
-                d,
-                semaphore=semaphore,
-                # chunk_size=chunk_size, #TODO add with resumaple api
-                # check_hash=check_hash, #TODO add with resumaple api
-                progress_cb=progress_cb,
-                progress_cb_type=progress_cb_type,
+        try:
+            if semaphore is None:
+                semaphore = self._api.get_default_semaphore()
+            tasks = []
+            for src, dst in zip(src_paths, dst_paths):
+                task = asyncio.create_task(
+                    self.upload_async(
+                        team_id=team_id,
+                        src=src,
+                        dst=dst,
+                        semaphore=semaphore,
+                        # chunk_size=chunk_size, #TODO add with resumaple api
+                        # check_hash=check_hash, #TODO add with resumaple api
+                        progress_cb=progress_cb,
+                        progress_cb_type=progress_cb_type,
+                    )
+                )
+                tasks.append(task)
+            for task in tasks:
+                await task
+        except Exception as e:
+            if enable_fallback:
+                logger.warning(
+                    f"Upload files bulk asynchronously failed. Fallback to synchronous upload.",
+                    exc_info=True,
+                )
+                if progress_cb is not None and progress_cb_type == "number":
+                    logger.warning(
+                        "Progress callback type 'number' is not supported for synchronous upload. "
+                        "Progress callback will be disabled."
+                    )
+                    progress_cb = None
+                self.upload_bulk(
+                    team_id=team_id,
+                    src_paths=src_paths,
+                    dst_paths=dst_paths,
+                    progress_cb=progress_cb,
+                )
+            else:
+                raise e
+
+    async def upload_directory_async(
+        self,
+        team_id: int,
+        local_dir: str,
+        remote_dir: str,
+        change_name_if_conflict: Optional[bool] = True,
+        progress_size_cb: Optional[Union[tqdm, Callable]] = None,
+        replace_if_conflict: Optional[bool] = False,
+        enable_fallback: Optional[bool] = True,
+    ) -> str:
+        """
+        Upload Directory to Team Files from local path.
+        Files are uploaded asynchronously.
+
+        :param team_id: Team ID in Supervisely.
+        :type team_id: int
+        :param local_dir: Path to local Directory.
+        :type local_dir: str
+        :param remote_dir: Path to Directory in Team Files.
+        :type remote_dir: str
+        :param change_name_if_conflict: Checks if given name already exists and adds suffix to the end of the name.
+        :type change_name_if_conflict: bool, optional
+        :param progress_size_cb: Function for tracking download progress.
+        :type progress_size_cb: Progress, optional
+        :param replace_if_conflict: If True, replace existing dir.
+        :type replace_if_conflict: bool, optional
+        :param enable_fallback: If True, the method will fallback to synchronous upload if an error occurs.
+        :type enable_fallback: bool, optional
+        :return: Path to Directory in Team Files
+        :rtype: :class:`str`
+        :Usage example:
+
+         .. code-block:: python
+
+            import supervisely as sly
+
+            os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+            os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+            api = sly.Api.from_env()
+
+            path_to_dir = "/My_App_Test/ds1"
+            local_path = "/home/admin/Downloads/My_local_test"
+
+            api.file.upload_directory(9, local_path, path_to_dir)
+        """
+        try:
+            if not remote_dir.startswith("/"):
+                remote_dir = "/" + remote_dir
+
+            if self.dir_exists(team_id, remote_dir):
+                if change_name_if_conflict is True:
+                    res_remote_dir = self.get_free_dir_name(team_id, remote_dir)
+                elif replace_if_conflict is True:
+                    res_remote_dir = remote_dir
+                else:
+                    raise FileExistsError(
+                        f"Directory {remote_dir} already exists in your team (id={team_id})"
+                    )
+            else:
+                res_remote_dir = remote_dir
+
+            local_files = await list_files_recursively_async(local_dir)
+            dir_prefix = local_dir.rstrip("/") + "/"
+            remote_files = [
+                res_remote_dir.rstrip("/") + "/" + file[len(dir_prefix) :] for file in local_files
+            ]
+
+            await self.upload_bulk_async(
+                team_id=team_id,
+                src_paths=local_files,
+                dst_paths=remote_files,
+                progress_cb=progress_size_cb,
             )
-            tasks.append(task)
-        await asyncio.gather(*tasks)
+        except Exception as e:
+            if enable_fallback:
+                logger.warning(
+                    f"Upload directory asynchronously failed. Fallback to synchronous upload.",
+                    exc_info=True,
+                )
+                res_remote_dir = self.upload_directory(
+                    team_id=team_id,
+                    local_dir=local_dir,
+                    remote_dir=res_remote_dir,
+                    change_name_if_conflict=change_name_if_conflict,
+                    progress_size_cb=progress_size_cb,
+                    replace_if_conflict=replace_if_conflict,
+                )
+            else:
+                raise e
+        return res_remote_dir

{supervisely-6.73.319 → supervisely-6.73.321}/supervisely/api/task_api.py

@@ -818,14 +818,21 @@ class TaskApi(ModuleApiBase, ModuleWithStatus):
         pass
 
     def set_output_project(
-        self, task_id: int, project_id: int, project_name: Optional[str] = None
+        self,
+        task_id: int,
+        project_id: int,
+        project_name: Optional[str] = None,
+        project_preview: Optional[str] = None,
     ) -> Dict:
         """set_output_project"""
         if project_name is None:
             project = self._api.project.get_info_by_id(project_id, raise_error=True)
             project_name = project.name
+            project_preview = project.image_preview_url
 
         output = {ApiField.PROJECT: {ApiField.ID: project_id, ApiField.TITLE: project_name}}
+        if project_preview is not None:
+            output[ApiField.PROJECT][ApiField.PREVIEW] = project_preview
         resp = self._api.post(
             "tasks.output.set", {ApiField.TASK_ID: task_id, ApiField.OUTPUT: output}
         )
@@ -1159,9 +1166,7 @@ class TaskApi(ModuleApiBase, ModuleWithStatus):
             )
         self._api.post("tasks.status.update", {ApiField.ID: task_id, ApiField.STATUS: status})
 
-    def set_output_experiment(
-        self, task_id: int, experiment_info: dict, project_name: str = None
-    ) -> Dict:
+    def set_output_experiment(self, task_id: int, experiment_info: dict) -> Dict:
         """
         Sets output for the task with experiment info.
 
@@ -1216,15 +1221,7 @@ class TaskApi(ModuleApiBase, ModuleWithStatus):
                 },
             }
         """
-        project_id = experiment_info.get("project_id")
-        if project_id is None:
-            raise ValueError("Key 'project_id' is required in experiment_info")
-        if project_name is None:
-            project = self._api.project.get_info_by_id(project_id, raise_error=True)
-            project_name = project.name
-
         output = {
-            ApiField.PROJECT: {ApiField.ID: project_id, ApiField.TITLE: project_name},
             ApiField.EXPERIMENT: {ApiField.DATA: {**experiment_info}},
         }
         resp = self._api.post(

{supervisely-6.73.319 → supervisely-6.73.321}/supervisely/app/fastapi/subapp.py

@@ -27,6 +27,7 @@ from fastapi import (
 )
 from fastapi.exception_handlers import http_exception_handler
 from fastapi.responses import JSONResponse
+from fastapi.routing import APIRouter
 from fastapi.staticfiles import StaticFiles
 
 import supervisely.io.env as sly_env
@@ -47,7 +48,7 @@ from supervisely.app.singleton import Singleton
 from supervisely.app.widgets_context import JinjaWidgets
 from supervisely.geometry.bitmap import Bitmap
 from supervisely.io.fs import dir_exists, mkdir
-from supervisely.sly_logger import logger
+from supervisely.sly_logger import create_formatter, logger
 
 # from supervisely.app.fastapi.request import Request
 
@@ -56,7 +57,9 @@ if TYPE_CHECKING:
 
 import logging
 
-uvicorn_logger = logging.getLogger("uvicorn.access")
+SUPERVISELY_SERVER_PATH_PREFIX = sly_env.supervisely_server_path_prefix()
+if SUPERVISELY_SERVER_PATH_PREFIX and not SUPERVISELY_SERVER_PATH_PREFIX.startswith("/"):
+    SUPERVISELY_SERVER_PATH_PREFIX = f"/{SUPERVISELY_SERVER_PATH_PREFIX}"
 
 
 class ReadyzFilter(logging.Filter):
@@ -67,8 +70,24 @@ class ReadyzFilter(logging.Filter):
         return True
 
 
-# Apply the filter
-uvicorn_logger.addFilter(ReadyzFilter())
+def _init_uvicorn_logger():
+    uvicorn_logger = logging.getLogger("uvicorn.access")
+    for handler in uvicorn_logger.handlers:
+        handler.setFormatter(create_formatter())
+    uvicorn_logger.addFilter(ReadyzFilter())
+
+
+_init_uvicorn_logger()
+
+
+class PrefixRouter(APIRouter):
+    def add_api_route(self, path, *args, **kwargs):
+        allowed_paths = ["/livez", "/is_alive", "/is_running", "/readyz", "/is_ready"]
+        if path in allowed_paths:
+            super().add_api_route(path, *args, **kwargs)
+        if SUPERVISELY_SERVER_PATH_PREFIX:
+            path = SUPERVISELY_SERVER_PATH_PREFIX + path
+        super().add_api_route(path, *args, **kwargs)
 
 
 class Event:
@@ -864,6 +883,7 @@ def _init(
 class _MainServer(metaclass=Singleton):
     def __init__(self):
         self._server = FastAPI()
+        self._server.router = PrefixRouter()
 
     def get_server(self) -> FastAPI:
         return self._server

{supervisely-6.73.319 → supervisely-6.73.321}/supervisely/io/env.py

@@ -554,3 +554,19 @@ def semaphore_size() -> int:
         postprocess_fn=lambda x: int(x),
         raise_not_found=False,
     )
+
+
+def supervisely_server_path_prefix() -> str:
+    """Returns routes prefix from environment variable using following
+        - SUPERVISELY_SERVER_PATH_PREFIX
+
+    :return: routes prefix
+    :rtype: str
+    """
+    return _parse_from_env(
+        name="supervisely_server_path_prefix",
+        keys=["SUPERVISELY_SERVER_PATH_PREFIX"],
+        postprocess_fn=lambda x: x,
+        default="",
+        raise_not_found=False,
+    )

{supervisely-6.73.319 → supervisely-6.73.321}/supervisely/io/fs.py

@@ -205,15 +205,19 @@ def list_files_recursively(
             for filename in file_names:
                 yield os.path.join(dir_name, filename)
 
-    valid_extensions = valid_extensions if ignore_valid_extensions_case is False else [ext.lower() for ext in valid_extensions]
+    valid_extensions = (
+        valid_extensions
+        if ignore_valid_extensions_case is False
+        else [ext.lower() for ext in valid_extensions]
+    )
     files = []
     for file_path in file_path_generator():
         file_ext = get_file_ext(file_path)
         if ignore_valid_extensions_case:
             file_ext.lower()
-        if (
-            valid_extensions is None or file_ext in valid_extensions
-        ) and (filter_fn is None or filter_fn(file_path)):
+        if (valid_extensions is None or file_ext in valid_extensions) and (
+            filter_fn is None or filter_fn(file_path)
+        ):
             files.append(file_path)
     return files
 
@@ -1558,3 +1562,76 @@ async def touch_async(path: str) -> None:
     async with aiofiles.open(path, "a"):
         loop = get_or_create_event_loop()
         await loop.run_in_executor(None, os.utime, path, None)
+
+
+async def list_files_recursively_async(
+    dir_path: str,
+    valid_extensions: Optional[List[str]] = None,
+    filter_fn: Optional[Callable[[str], bool]] = None,
+    ignore_valid_extensions_case: bool = False,
+) -> List[str]:
+    """
+    Recursively list files in the directory asynchronously.
+    Returns list with all file paths.
+    Can be filtered by valid extensions and filter function.
+
+    :param dir_path: Target directory path.
+    :type dir_path: str
+    :param valid_extensions: List of valid extensions. Default is None.
+    :type valid_extensions: Optional[List[str]]
+    :param filter_fn: Filter function. Default is None.
+    :type filter_fn: Optional[Callable[[str], bool]]
+    :param ignore_valid_extensions_case: Ignore case when checking valid extensions. Default is False.
+    :type ignore_valid_extensions_case: bool
+    :returns: List of file paths
+    :rtype: List[str]
+
+    :Usage example:
+    
+         .. code-block:: python
+    
+            import supervisely as sly
+    
+            dir_path = '/home/admin/work/projects/examples'
+            loop = sly.utils.get_or_create_event_loop()
+            coro = sly.fs.list_files_recursively_async(dir_path)
+            if loop.is_running():
+                future = asyncio.run_coroutine_threadsafe(coro, loop)
+                files = future.result()
+            else:
+                files = loop.run_until_complete(coro)
+    """
+
+    def sync_file_list():
+        if valid_extensions and ignore_valid_extensions_case:
+            valid_extensions_set = set(map(str.lower, valid_extensions))
+        else:
+            valid_extensions_set = set(valid_extensions) if valid_extensions else None
+
+        files = []
+        for dir_name, _, file_names in os.walk(dir_path):
+            full_paths = [os.path.join(dir_name, filename) for filename in file_names]
+
+            if valid_extensions_set:
+                full_paths = [
+                    fp
+                    for fp in full_paths
+                    if (
+                        ext := (
+                            os.path.splitext(fp)[1].lower()
+                            if ignore_valid_extensions_case
+                            else os.path.splitext(fp)[1]
+                        )
+                    )
+                    in valid_extensions_set
+                ]
+
+            if filter_fn:
+                full_paths = [fp for fp in full_paths if filter_fn(fp)]
+
+            files.extend(full_paths)
+
+        return files
+
+    loop = get_or_create_event_loop()
+    return await loop.run_in_executor(None, sync_file_list)