supervisely 6.73.220__py3-none-any.whl → 6.73.222__py3-none-any.whl

supervisely/convert/image/sly/sly_image_converter.py

@@ -2,21 +2,21 @@ import os
 from typing import Dict, Optional
 
 import supervisely.convert.image.sly.sly_image_helper as sly_image_helper
-from supervisely.convert.image.image_helper import validate_image_bounds
 from supervisely import (
     Annotation,
     Dataset,
+    Label,
     OpenMode,
     Project,
     ProjectMeta,
     Rectangle,
-    Label,
     logger,
 )
 from supervisely._utils import generate_free_name
 from supervisely.api.api import Api
 from supervisely.convert.base_converter import AvailableImageConverters
 from supervisely.convert.image.image_converter import ImageConverter
+from supervisely.convert.image.image_helper import validate_image_bounds
 from supervisely.io.fs import dirs_filter, file_exists, get_file_ext
 from supervisely.io.json import load_json_file
 from supervisely.project.project import find_project_dirs
@@ -31,6 +31,7 @@ class SLYImageConverter(ImageConverter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._project_structure = None
+        self._supports_links = True
 
     def __str__(self):
         return AvailableImageConverters.SLY
@@ -74,6 +75,8 @@ class SLYImageConverter(ImageConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links and self._supports_links:
+            self._download_remote_ann_files()
         if self.read_sly_project(self._input_data):
             return True
 
@@ -136,6 +139,8 @@ class SLYImageConverter(ImageConverter):
             meta = self._meta
 
         if item.ann_data is None:
+            if self._upload_as_links:
+                item.set_shape([None, None])
             return item.create_empty_annotation()
 
         try:
@@ -151,7 +156,7 @@ class SLYImageConverter(ImageConverter):
             )
             return Annotation.from_json(ann_json, meta).clone(labels=labels)
         except Exception as e:
-            logger.warn(f"Failed to convert annotation: {repr(e)}")
+            logger.warning(f"Failed to convert annotation: {repr(e)}")
             return item.create_empty_annotation()
 
     def read_sly_project(self, input_data: str) -> bool:
@@ -163,7 +168,7 @@ class SLYImageConverter(ImageConverter):
             logger.debug("Trying to find Supervisely project format in the input data")
             project_dirs = [d for d in find_project_dirs(input_data)]
             if len(project_dirs) > 1:
-                logger.info("Found multiple Supervisely projects")
+                logger.info("Found multiple possible Supervisely projects in the input data")
             meta = None
             for project_dir in project_dirs:
                 project_fs = Project(project_dir, mode=OpenMode.READ)

supervisely/convert/pointcloud_episodes/sly/sly_pointcloud_episodes_converter.py

@@ -71,11 +71,17 @@ class SLYPointcloudEpisodesConverter(PointcloudEpisodeConverter):
                     continue
 
                 ext = get_file_ext(full_path)
+                recognized_ext = imghdr.what(full_path)
                 if file in JUNK_FILES:
                     continue
                 elif ext in self.ann_ext:
                     rimg_json_dict[file] = full_path
-                elif imghdr.what(full_path):
+                elif recognized_ext:
+                    if ext.lower() == ".pcd":
+                        logger.warning(
+                            f"File '{file}' has been recognized as '.{recognized_ext}' format. Skipping."
+                        )
+                        continue
                     rimg_dict[file] = full_path
                     if ext not in used_img_ext:
                         used_img_ext.append(ext)

supervisely/convert/video/sly/sly_video_converter.py

@@ -12,6 +12,10 @@ from supervisely.video.video import validate_ext as validate_video_ext
 
 class SLYVideoConverter(VideoConverter):
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._supports_links = True
+
     def __str__(self) -> str:
         return AvailableVideoConverters.SLY
 
@@ -45,6 +49,8 @@ class SLYVideoConverter(VideoConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links and self._supports_links:
+            self._download_remote_ann_files()
         detected_ann_cnt = 0
         videos_list, ann_dict = [], {}
         for root, _, files in os.walk(self._input_data):
@@ -103,6 +109,8 @@ class SLYVideoConverter(VideoConverter):
             meta = self._meta
 
         if item.ann_data is None:
+            if self._upload_as_links:
+                return None
             return item.create_empty_annotation()
 
         try:
@@ -113,5 +121,5 @@ class SLYVideoConverter(VideoConverter):
                 ann_json = sly_video_helper.rename_in_json(ann_json, renamed_classes, renamed_tags)
             return VideoAnnotation.from_json(ann_json, meta)
         except Exception as e:
-            logger.warn(f"Failed to convert annotation: {repr(e)}")
+            logger.warning(f"Failed to convert annotation: {repr(e)}")
             return item.create_empty_annotation()

supervisely/convert/video/video_converter.py

@@ -1,6 +1,6 @@
 import os
 import subprocess
-from typing import Dict, Optional, Union
+from typing import Dict, Optional, Tuple, Union
 
 import cv2
 import magic
@@ -57,10 +57,22 @@ class VideoConverter(BaseConverter):
                 self._frame_count = frame_count
             self._custom_data = custom_data if custom_data is not None else {}
 
+        @property
+        def shape(self) -> Tuple[int, int]:
+            return self._shape
+
+        @shape.setter
+        def shape(self, shape: Optional[Tuple[int, int]] = None):
+            self._shape = shape if shape is not None else [None, None]
+
         @property
         def frame_count(self) -> int:
             return self._frame_count
 
+        @frame_count.setter
+        def frame_count(self, frame_count: int):
+            self._frame_count = frame_count
+
         @property
         def name(self) -> str:
             if self._name is not None:
@@ -75,11 +87,11 @@ class VideoConverter(BaseConverter):
             return VideoAnnotation(self._shape, self._frame_count)
 
     def __init__(
-            self,
-            input_data: str,
-            labeling_interface: Optional[Union[LabelingInterface, str]],
-            upload_as_links: bool,
-            remote_files_map: Optional[Dict[str, str]] = None,
+        self,
+        input_data: str,
+        labeling_interface: Optional[Union[LabelingInterface, str]],
+        upload_as_links: bool,
+        remote_files_map: Optional[Dict[str, str]] = None,
     ):
         super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
         self._key_id_map: KeyIdMap = None
@@ -114,7 +126,9 @@ class VideoConverter(BaseConverter):
         existing_names = set([vid.name for vid in api.video.get_list(dataset_id)])
 
         # check video codecs, mimetypes and convert if needed
-        convert_progress, convert_progress_cb = self.get_progress(self.items_count, "Preparing videos...")
+        convert_progress, convert_progress_cb = self.get_progress(
+            self.items_count, "Preparing videos..."
+        )
         for item in self._items:
             item_name, item_path = self.convert_to_mp4_if_needed(item.path)
             item.name = item_name
@@ -124,11 +138,14 @@ class VideoConverter(BaseConverter):
             convert_progress.close()
 
         has_large_files = False
+        size_progress_cb = None
         progress_cb, progress, ann_progress, ann_progress_cb = None, None, None, None
         if log_progress and not self.upload_as_links:
             progress, progress_cb = self.get_progress(self.items_count, "Uploading videos...")
             file_sizes = [get_file_size(item.path) for item in self._items]
-            has_large_files = any([self._check_video_file_size(file_size) for file_size in file_sizes])
+            has_large_files = any(
+                [self._check_video_file_size(file_size) for file_size in file_sizes]
+            )
             if has_large_files:
                 upload_progress = []
                 size_progress_cb = self._get_video_upload_progress(upload_progress)
@@ -146,17 +163,19 @@ class VideoConverter(BaseConverter):
                 item_paths.append(item.path)
                 item_names.append(item.name)
 
-                if not self.upload_as_links:
-                    # TODO: implement generating annotations for remote videos
+                ann = None
+                if not self.upload_as_links or self.supports_links:
                     ann = self.to_supervisely(item, meta, renamed_classes, renamed_tags)
-                    figures_cnt += len(ann.figures)
-                    anns.append(ann)
+                    if ann is not None:
+                        figures_cnt += len(ann.figures)
+                anns.append(ann)
 
             if self.upload_as_links:
                 vid_infos = api.video.upload_links(
                     dataset_id,
                     item_paths,
                     item_names,
+                    skip_download=True,
                 )
             else:
                 vid_infos = api.video.upload_paths(
@@ -164,22 +183,24 @@ class VideoConverter(BaseConverter):
                     item_names,
                     item_paths,
                     progress_cb=progress_cb if log_progress else None,
-                    item_progress=size_progress_cb if log_progress and has_large_files else None, # pylint: disable=used-before-assignment
+                    item_progress=(size_progress_cb if log_progress and has_large_files else None),
                 )
-                vid_ids = [vid_info.id for vid_info in vid_infos]
+            vid_ids = [vid_info.id for vid_info in vid_infos]
 
-                if log_progress and has_large_files and figures_cnt > 0:
-                    ann_progress, ann_progress_cb = self.get_progress(figures_cnt, "Uploading annotations...")
+            if log_progress and has_large_files and figures_cnt > 0:
+                ann_progress, ann_progress_cb = self.get_progress(
+                    figures_cnt, "Uploading annotations..."
+                )
 
-                for video_id, ann in zip(vid_ids, anns):
-                    if ann is None:
-                        ann = VideoAnnotation(item.shape, item.frame_count)
-                    api.video.annotation.append(video_id, ann, progress_cb=ann_progress_cb)
+            for vid, ann, item, info in zip(vid_ids, anns, batch, vid_infos):
+                if ann is None:
+                    ann = VideoAnnotation((info.frame_height, info.frame_width), info.frames_count)
+                api.video.annotation.append(vid, ann, progress_cb=ann_progress_cb)
 
         if log_progress and is_development():
-            if progress is not None: # pylint: disable=possibly-used-before-assignment
+            if progress is not None:
                 progress.close()
-            if not self.upload_as_links and ann_progress is not None:
+            if ann_progress is not None:
                 ann_progress.close()
         logger.info(f"Dataset ID:{dataset_id} has been successfully uploaded.")
 
@@ -268,7 +289,7 @@ class VideoConverter(BaseConverter):
         )
 
     def _check_video_file_size(self, file_size):
-        return file_size > 20 * 1024 * 1024 # 20 MB
+        return file_size > 20 * 1024 * 1024  # 20 MB
 
     def _get_video_upload_progress(self, upload_progress):
         upload_progress = []

supervisely/io/fs.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 
 # docs
+import asyncio
 import errno
 import mimetypes
 import os
@@ -10,6 +11,7 @@ import subprocess
 import tarfile
 from typing import Callable, Dict, Generator, List, Literal, Optional, Tuple, Union
 
+import aiofiles
 import requests
 from requests.structures import CaseInsensitiveDict
 from tqdm import tqdm
@@ -1348,3 +1350,126 @@ def str_is_url(string: str) -> bool:
         return all([result.scheme, result.netloc])
     except ValueError:
         return False
+
+
+async def copy_file_async(
+    src: str,
+    dst: str,
+    progress_cb: Optional[Union[tqdm, Callable]] = None,
+    progress_cb_type: Literal["number", "size"] = "size",
+) -> None:
+    """
+    Asynchronously copy file from one path to another, if destination directory doesn't exist it will be created.
+
+    :param src: Source file path.
+    :type src: str
+    :param dst: Destination file path.
+    :type dst: str
+    :param progress_cb: Function for tracking copy progress.
+    :type progress_cb: Union[tqdm, Callable], optional
+    :param progress_cb_type: Type of progress callback. Can be "number" or "size". Default is "size".
+    :type progress_cb_type: Literal["number", "size"], optional
+    :returns: None
+    :rtype: :class:`NoneType`
+    :Usage example:
+
+     .. code-block:: python
+
+        from supervisely.io.fs import async_copy_file
+        await async_copy_file('/home/admin/work/projects/example/1.png', '/home/admin/work/tests/2.png')
+    """
+    ensure_base_path(dst)
+    async with aiofiles.open(dst, "wb") as out_f:
+        async with aiofiles.open(src, "rb") as in_f:
+            while True:
+                chunk = await in_f.read(1024 * 1024)
+                if not chunk:
+                    break
+                await out_f.write(chunk)
+                if progress_cb is not None and progress_cb_type == "size":
+                    progress_cb(len(chunk))
+    if progress_cb is not None and progress_cb_type == "number":
+        progress_cb(1)
+
+
+async def get_file_hash_async(path: str) -> str:
+    """
+    Get hash from target file asynchronously.
+
+    :param path: Target file path.
+    :type path: str
+    :returns: File hash
+    :rtype: :class:`str`
+    :Usage example:
+
+     .. code-block:: python
+
+        from supervisely.io.fs import get_file_hash_async
+        hash = await get_file_hash_async('/home/admin/work/projects/examples/1.jpeg') # rKLYA/p/P64dzidaQ/G7itxIz3ZCVnyUhEE9fSMGxU4=
+    """
+    async with aiofiles.open(path, "rb") as file:
+        file_bytes = await file.read()
+        return get_bytes_hash(file_bytes)
+
+
+async def unpack_archive_async(
+    archive_path: str, target_dir: str, remove_junk=True, is_split=False, chunk_size_mb: int = 50
+) -> None:
+    """
+    Unpacks archive to the target directory, removes junk files and directories.
+    To extract a split archive, you must pass the path to the first part in archive_path. Archive parts must be in the same directory. Format: archive_name.tar.001, archive_name.tar.002, etc. Works with tar and zip.
+    You can adjust the size of the chunk to read from the file, while unpacking the file from parts.
+    Be careful with this parameter, it can affect the performance of the function.
+
+    :param archive_path: Path to the archive.
+    :type archive_path: str
+    :param target_dir: Path to the target directory.
+    :type target_dir: str
+    :param remove_junk: Remove junk files and directories. Default is True.
+    :type remove_junk: bool
+    :param is_split: Determines if the source archive is split into parts. If True, archive_path must be the path to the first part. Default is False.
+    :type is_split: bool
+    :param chunk_size_mb: Size of the chunk to read from the file. Default is 50Mb.
+    :type chunk_size_mb: int
+    :returns: None
+    :rtype: :class:`NoneType`
+    :Usage example:
+
+     .. code-block:: python
+
+        import supervisely as sly
+
+        archive_path = '/home/admin/work/examples.tar'
+        target_dir = '/home/admin/work/projects'
+
+        await sly.fs.unpack_archive(archive_path, target_dir)
+    """
+    if is_split:
+        chunk = chunk_size_mb * 1024 * 1024
+        base_name = get_file_name(archive_path)
+        dir_name = os.path.dirname(archive_path)
+        if get_file_ext(base_name) in (".zip", ".tar"):
+            ext = get_file_ext(base_name)
+            base_name = get_file_name(base_name)
+        else:
+            ext = get_file_ext(archive_path)
+        parts = sorted([f for f in os.listdir(dir_name) if f.startswith(base_name)])
+        combined = os.path.join(dir_name, f"combined{ext}")
+
+        async with aiofiles.open(combined, "wb") as output_file:
+            for part in parts:
+                part_path = os.path.join(dir_name, part)
+                async with aiofiles.open(part_path, "rb") as input_file:
+                    while True:
+                        data = await input_file.read(chunk)
+                        if not data:
+                            break
+                        await output_file.write(data)
+        archive_path = combined
+
+    loop = asyncio.get_running_loop()
+    await loop.run_in_executor(None, shutil.unpack_archive, archive_path, target_dir)
+    if is_split:
+        silent_remove(archive_path)
+    if remove_junk:
+        remove_junk_from_dir(target_dir)

supervisely/io/fs_cache.py

@@ -1,8 +1,8 @@
 # coding: utf-8
 
+import hashlib
 import os
 import os.path as osp
-import hashlib
 import shutil
 
 from supervisely.io import fs as sly_fs
@@ -141,6 +141,24 @@ class FileCache(FSCache):
     def _rm_obj_impl(self, st_path):
         os.remove(st_path)
 
+    async def _read_obj_impl_async(self, st_path, dst_path):
+        sly_fs.ensure_base_path(dst_path)
+        await sly_fs.copy_file_async(st_path, dst_path)
+
+    async def write_object_async(self, src_path, data_hash):
+        suffix = self._get_suffix(src_path)
+        st_path = self.get_storage_path(data_hash, suffix)
+        if not self._storage_obj_exists(st_path, suffix):
+            await sly_fs.copy_file_async(src_path, st_path)
+
+    async def read_object_async(self, data_hash, dst_path):
+        suffix = self._get_suffix(dst_path)
+        st_path = self.check_storage_object(data_hash, suffix)
+        if not st_path:
+            return None
+        await self._read_obj_impl(st_path, dst_path)
+        return dst_path
+
 
 class NNCache(FSCache):
     def _storage_obj_exists(self, st_path, suffix):

supervisely/io/network_exceptions.py

@@ -3,6 +3,7 @@
 import time
 import traceback
 
+import httpx
 import requests
 
 CONNECTION_ERROR = "Temporary connection error, please wait ..."
@@ -44,18 +45,22 @@ def process_requests_exception(
             requests.exceptions.Timeout,
             requests.exceptions.TooManyRedirects,
             requests.exceptions.ChunkedEncodingError,
+            httpx.ConnectError,
+            httpx.TimeoutException,
+            httpx.TooManyRedirects,
+            httpx.ProtocolError,
         ),
     )
 
     is_server_retryable_error = (
-        isinstance(exc, requests.exceptions.HTTPError)
+        isinstance(exc, (requests.exceptions.HTTPError, httpx.HTTPStatusError))
         and hasattr(exc, "response")
         and (exc.response.status_code in RETRY_STATUS_CODES)
     )
 
     is_need_ping_error = False
     if (
-        isinstance(exc, requests.exceptions.HTTPError)
+        isinstance(exc, (requests.exceptions.HTTPError, httpx.HTTPStatusError))
         and hasattr(exc, "response")
         and (exc.response.status_code == 400)
     ):
@@ -91,7 +96,7 @@ def process_requests_exception(
         )
     elif response is None:
         process_unhandled_request(external_logger, exc)
-    elif isinstance(exc, requests.exceptions.HTTPError):
+    elif isinstance(exc, (requests.exceptions.HTTPError, httpx.HTTPStatusError)):
         process_invalid_request(external_logger, exc, response, verbose)
     else:
         process_unhandled_request(external_logger, exc)
@@ -125,6 +130,18 @@ def process_retryable_request(
 
 
 def process_invalid_request(external_logger, exc, response, verbose=True):
+    if type(response) in (httpx.Response, requests.Response):
+        reason = (
+            response.content.decode("utf-8")
+            if not hasattr(response, "is_stream_consumed")
+            else "Content is not acessible for streaming responses"
+        )
+        status_code = response.status_code
+        url = response.url
+    else:
+        reason = "Reason is unknown"
+        status_code = None
+        url = None
     if verbose:
         external_logger.warn(
             REQUEST_FAILED,

supervisely/task/progress.py

@@ -5,7 +5,7 @@ import inspect
 import math
 import re
 from functools import partial, wraps
-from typing import Optional, Union, Dict
+from typing import Dict, Optional, Union
 
 from tqdm import tqdm
 

{supervisely-6.73.220.dist-info → supervisely-6.73.222.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.220
+Version: 6.73.222
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely
@@ -68,6 +68,8 @@ Requires-Dist: cacheout==0.14.1
 Requires-Dist: jsonschema<=4.20.0,>=2.6.0
 Requires-Dist: pyjwt<3.0.0,>=2.1.0
 Requires-Dist: zstd
+Requires-Dist: aiofiles
+Requires-Dist: httpx[http2]==0.27.2
 Provides-Extra: apps
 Requires-Dist: uvicorn[standard]<1.0.0,>=0.18.2; extra == "apps"
 Requires-Dist: fastapi<1.0.0,>=0.79.0; extra == "apps"