supervisely 6.73.221__py3-none-any.whl → 6.73.223__py3-none-any.whl

supervisely/convert/base_converter.py

@@ -1,15 +1,24 @@
 from __future__ import annotations
 
 import os
+from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
 from tqdm import tqdm
 
-from supervisely._utils import is_production
+from supervisely._utils import batched, is_production
 from supervisely.annotation.annotation import Annotation
 from supervisely.annotation.tag_meta import TagValueType
 from supervisely.api.api import Api
-from supervisely.io.fs import get_file_ext, get_file_name_with_ext
+from supervisely.io.env import team_id
+from supervisely.io.fs import (
+    get_file_ext,
+    get_file_name_with_ext,
+    is_archive,
+    remove_dir,
+    silent_remove,
+    unpack_archive,
+)
 from supervisely.project.project_meta import ProjectMeta
 from supervisely.project.project_settings import LabelingInterface
 from supervisely.sly_logger import logger
@@ -145,12 +154,16 @@ class BaseConverter:
         remote_files_map: Optional[Dict[str, str]] = None,
     ):
         self._input_data: str = input_data
-        self._items: List[self.BaseItem] = []
+        self._items: List[BaseConverter.BaseItem] = []
         self._meta: ProjectMeta = None
         self._labeling_interface = labeling_interface or LabelingInterface.DEFAULT
+
+        # import as links settings
         self._upload_as_links: bool = upload_as_links
         self._remote_files_map: Optional[Dict[str, str]] = remote_files_map
         self._supports_links = False  # if converter supports uploading by links
+        self._api = Api.from_env() if self._upload_as_links else None
+        self._team_id = team_id() if self._upload_as_links else None
         self._converter = None
 
         if self._labeling_interface not in LabelingInterface.values():
@@ -419,3 +432,81 @@ class BaseConverter:
 
             return meta1.clone(project_settings=new_settings)
         return meta1
+
+    def _download_remote_ann_files(self) -> None:
+        """
+        Download all annotation files from Cloud Storage to the local storage.
+        Needed to detect annotation format if "upload_as_links" is enabled.
+        """
+        if not self.upload_as_links:
+            return
+
+        ann_archives = {l: r for l, r in self._remote_files_map.items() if is_archive(l)}
+
+        anns_to_download = {
+            l: r for l, r in self._remote_files_map.items() if get_file_ext(l) == self.ann_ext
+        }
+        if not anns_to_download and not ann_archives:
+            return
+
+        import asyncio
+
+        for files_type, files in {
+            "annotations": anns_to_download,
+            "archives": ann_archives,
+        }.items():
+            if not files:
+                continue
+
+            is_archive_type = files_type == "archives"
+
+            file_size = None
+            if is_archive_type:
+                logger.info(f"Remote archives detected.")
+                file_size = sum(
+                    self._api.storage.get_info_by_path(self._team_id, remote_path).sizeb
+                    for remote_path in files.values()
+                )
+
+            loop = asyncio.get_event_loop()
+            _, progress_cb = self.get_progress(
+                len(files) if not is_archive_type else file_size,
+                f"Downloading {files_type} from remote storage",
+                is_size=is_archive_type,
+            )
+
+            for local_path in files.keys():
+                silent_remove(local_path)
+
+            logger.info(f"Downloading {files_type} from remote storage...")
+            loop.run_until_complete(
+                self._api.storage.download_bulk_async(
+                    team_id=self._team_id,
+                    remote_paths=list(files.values()),
+                    local_save_paths=list(files.keys()),
+                    progress_cb=progress_cb,
+                    progress_cb_type="number" if not is_archive_type else "size",
+                )
+            )
+            logger.info("Possible annotations downloaded successfully.")
+
+            if is_archive_type:
+                for local_path in files.keys():
+                    parent_dir = Path(local_path).parent
+                    if parent_dir.name == "ann":
+                        target_dir = parent_dir
+                    else:
+                        target_dir = parent_dir / "ann"
+                        target_dir.mkdir(parents=True, exist_ok=True)
+
+                    unpack_archive(local_path, str(target_dir))
+                    silent_remove(local_path)
+
+                    dirs = [d for d in target_dir.iterdir() if d.is_dir()]
+                    files = [f for f in target_dir.iterdir() if f.is_file()]
+                    if len(dirs) == 1 and len(files) == 0:
+                        for file in dirs[0].iterdir():
+                            file.rename(target_dir / file.name)
+                        remove_dir(str(dirs[0]))
+
+                    logger.info(f"Archive {local_path} unpacked successfully to {str(target_dir)}")

supervisely/convert/converter.py

@@ -1,15 +1,16 @@
 import os
 from pathlib import Path
+from typing import Optional
 
 from tqdm import tqdm
 
-from typing import Literal, Optional
-
 from supervisely._utils import is_production
 from supervisely.api.api import Api
 from supervisely.app import get_data_dir
 from supervisely.convert.image.csv.csv_converter import CSVConverter
-from supervisely.convert.image.high_color.high_color_depth import HighColorDepthImageConverter
+from supervisely.convert.image.high_color.high_color_depth import (
+    HighColorDepthImageConverter,
+)
 from supervisely.convert.image.image_converter import ImageConverter
 from supervisely.convert.pointcloud.pointcloud_converter import PointcloudConverter
 from supervisely.convert.pointcloud_episodes.pointcloud_episodes_converter import (
@@ -28,10 +29,10 @@ from supervisely.io.fs import (
     touch,
     unpack_archive,
 )
+from supervisely.project.project_settings import LabelingInterface
 from supervisely.project.project_type import ProjectType
 from supervisely.sly_logger import logger
 from supervisely.task.progress import Progress
-from supervisely.project.project_settings import LabelingInterface
 
 
 class ImportManager:
@@ -164,7 +165,7 @@ class ImportManager:
         dir_path = remote_path.rstrip("/") if is_dir else os.path.dirname(remote_path)
         dir_name = os.path.basename(dir_path)
 
-        local_path = os.path.join(get_data_dir(), dir_name)
+        local_path = os.path.abspath(os.path.join(get_data_dir(), dir_name))
         mkdir(local_path, remove_content_if_exists=True)
 
         if is_dir:
@@ -172,17 +173,22 @@ class ImportManager:
         else:
             files = [self._api.storage.get_info_by_path(self._team_id, remote_path)]
 
+        unique_directories = set()
         for file in files:
             new_path = file.path.replace(dir_path, local_path)
             self._remote_files_map[new_path] = file.path
             Path(new_path).parent.mkdir(parents=True, exist_ok=True)
+            unique_directories.add(str(Path(file.path).parent))
             touch(new_path)
 
+        logger.info(f"Scanned remote directories:\n   - " + "\n   - ".join(unique_directories))
         return local_path
 
     def _unpack_archives(self, local_path):
         """Unpack if input data contains an archive."""
 
+        if self._upload_as_links:
+            return
         new_paths_to_scan = [local_path]
         while len(new_paths_to_scan) > 0:
             archives = []

supervisely/convert/image/image_converter.py

@@ -114,6 +114,7 @@ class ImageConverter(BaseConverter):
         batch_size: int = 50,
         log_progress=True,
         entities: List[Item] = None,
+        progress_cb=None,
     ) -> None:
         """Upload converted data to Supervisely"""
         dataset_info = api.dataset.get_info_by_id(dataset_id, raise_error=True)
@@ -122,10 +123,14 @@ class ImageConverter(BaseConverter):
         meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
 
         existing_names = set([img.name for img in api.image.get_list(dataset_id)])
-        if log_progress:
-            progress, progress_cb = self.get_progress(self.items_count, "Uploading images...")
-        else:
-            progress_cb = None
+        progress = None
+        if progress_cb is not None:
+            log_progress = True
+        elif log_progress:
+            progress, progress_cb = self.get_progress(self.items_count, "Uploading")
+
+        if self.upload_as_links:
+            batch_size = 1000
 
         for batch in batched(entities or self._items, batch_size=batch_size):
             item_names = []
@@ -137,8 +142,8 @@ class ImageConverter(BaseConverter):
                 if item.path is None:
                     continue  # image has failed validation
                 item.name = f"{get_file_name(item.path)}{get_file_ext(item.path).lower()}"
-                if self.upload_as_links:
-                    ann = None  # TODO: implement
+                if self.upload_as_links and not self.supports_links:
+                    ann = None
                 else:
                     ann = self.to_supervisely(item, meta, renamed_classes, renamed_tags)
                 name = generate_free_name(
@@ -160,27 +165,40 @@ class ImageConverter(BaseConverter):
             with ApiContext(
                 api=api, project_id=project_id, dataset_id=dataset_id, project_meta=meta
             ):
-                upload_method = (
-                    api.image.upload_links if self.upload_as_links else api.image.upload_paths
-                )
-                img_infos = upload_method(
-                    dataset_id,
-                    item_names,
-                    item_paths,
-                    metas=item_metas,
-                    conflict_resolution="rename",
-                )
+                if self.upload_as_links:
+                    img_infos = api.image.upload_links(
+                        dataset_id,
+                        item_names,
+                        item_paths,
+                        metas=item_metas,
+                        batch_size=batch_size,
+                        conflict_resolution="rename",
+                        force_metadata_for_links=False,
+                    )
+                else:
+                    img_infos = api.image.upload_paths(
+                        dataset_id,
+                        item_names,
+                        item_paths,
+                        metas=item_metas,
+                        conflict_resolution="rename",
+                    )
+
                 img_ids = [img_info.id for img_info in img_infos]
                 if len(anns) == len(img_ids):
-                    api.annotation.upload_anns(img_ids, anns)
+                    api.annotation.upload_anns(
+                        img_ids, anns, skip_bounds_validation=self.upload_as_links
+                    )
 
             if log_progress:
                 progress_cb(len(batch))
 
         if log_progress:
-            if is_development():
+            if is_development() and progress is not None:
                 progress.close()
-        logger.info(f"Dataset ID:'{dataset_id}' has been successfully uploaded.")
+        logger.info(
+            f"Dataset has been successfully uploaded → {dataset_info.name}, ID:{dataset_id}"
+        )
 
     def validate_image(self, path: str) -> Tuple[str, str]:
         if self.upload_as_links:
@@ -188,6 +206,9 @@ class ImageConverter(BaseConverter):
         return image_helper.validate_image(path)
 
     def is_image(self, path: str) -> bool:
+        if self._upload_as_links and self.supports_links:
+            ext = get_file_ext(path)
+            return ext.lower() in self.allowed_exts
         mimetypes.add_type("image/heic", ".heic")  # to extend types_map
         mimetypes.add_type("image/heif", ".heif")  # to extend types_map
         mimetypes.add_type("image/jpeg", ".jfif")  # to extend types_map

supervisely/convert/image/sly/fast_sly_image_converter.py

@@ -20,6 +20,10 @@ from supervisely.convert.image.image_helper import validate_image_bounds
 
 class FastSlyImageConverter(SLYImageConverter, ImageConverter):
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._supports_links = False
+
     def validate_format(self) -> bool:
 
         detected_ann_cnt = 0

supervisely/convert/image/sly/sly_image_converter.py

@@ -2,21 +2,21 @@ import os
 from typing import Dict, Optional
 
 import supervisely.convert.image.sly.sly_image_helper as sly_image_helper
-from supervisely.convert.image.image_helper import validate_image_bounds
 from supervisely import (
     Annotation,
     Dataset,
+    Label,
     OpenMode,
     Project,
     ProjectMeta,
     Rectangle,
-    Label,
     logger,
 )
-from supervisely._utils import generate_free_name
+from supervisely._utils import generate_free_name, is_development
 from supervisely.api.api import Api
 from supervisely.convert.base_converter import AvailableImageConverters
 from supervisely.convert.image.image_converter import ImageConverter
+from supervisely.convert.image.image_helper import validate_image_bounds
 from supervisely.io.fs import dirs_filter, file_exists, get_file_ext
 from supervisely.io.json import load_json_file
 from supervisely.project.project import find_project_dirs
@@ -31,6 +31,7 @@ class SLYImageConverter(ImageConverter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._project_structure = None
+        self._supports_links = True
 
     def __str__(self):
         return AvailableImageConverters.SLY
@@ -74,6 +75,8 @@ class SLYImageConverter(ImageConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links and self._supports_links:
+            self._download_remote_ann_files()
         if self.read_sly_project(self._input_data):
             return True
 
@@ -136,6 +139,8 @@ class SLYImageConverter(ImageConverter):
             meta = self._meta
 
         if item.ann_data is None:
+            if self._upload_as_links:
+                item.set_shape([None, None])
             return item.create_empty_annotation()
 
         try:
@@ -151,7 +156,7 @@ class SLYImageConverter(ImageConverter):
             )
             return Annotation.from_json(ann_json, meta).clone(labels=labels)
         except Exception as e:
-            logger.warn(f"Failed to convert annotation: {repr(e)}")
+            logger.warning(f"Failed to convert annotation: {repr(e)}")
             return item.create_empty_annotation()
 
     def read_sly_project(self, input_data: str) -> bool:
@@ -163,7 +168,9 @@ class SLYImageConverter(ImageConverter):
             logger.debug("Trying to find Supervisely project format in the input data")
             project_dirs = [d for d in find_project_dirs(input_data)]
             if len(project_dirs) > 1:
-                logger.info("Found multiple Supervisely projects")
+                logger.info("Found multiple possible Supervisely projects in the input data")
+            else:
+                logger.info("Possible Supervisely project found in the input data")
             meta = None
             for project_dir in project_dirs:
                 project_fs = Project(project_dir, mode=OpenMode.READ)
@@ -276,6 +283,12 @@ class SLYImageConverter(ImageConverter):
         existing_datasets = api.dataset.get_list(project_id, recursive=True)
         existing_datasets = {ds.name for ds in existing_datasets}
 
+        if log_progress:
+            progress, progress_cb = self.get_progress(self.items_count, "Uploading project")
+        else:
+            progress, progress_cb = None, None
+
+        logger.info("Uploading project structure")
         def _upload_project(
             project_structure: Dict,
             project_id: int,
@@ -283,7 +296,6 @@ class SLYImageConverter(ImageConverter):
             parent_id: Optional[int] = None,
             first_dataset=False,
         ):
-
             for ds_name, value in project_structure.items():
                 ds_name = generate_free_name(existing_datasets, ds_name, extend_used_names=True)
                 if first_dataset:
@@ -293,13 +305,17 @@ class SLYImageConverter(ImageConverter):
                     dataset_id = api.dataset.create(project_id, ds_name, parent_id=parent_id).id
 
                 items = value.get(DATASET_ITEMS, [])
+                nested_datasets = value.get(NESTED_DATASETS, {})
+                logger.info(f"Dataset: {ds_name}, items: {len(items)}, nested datasets: {len(nested_datasets)}")
                 if items:
                     super(SLYImageConverter, self).upload_dataset(
-                        api, dataset_id, batch_size, log_progress, entities=items
+                        api, dataset_id, batch_size, entities=items, progress_cb=progress_cb
                     )
 
-                nested_datasets = value.get(NESTED_DATASETS, {})
                 if nested_datasets:
                     _upload_project(nested_datasets, project_id, dataset_id, dataset_id)
 
         _upload_project(self._project_structure, project_id, dataset_id, first_dataset=True)
+
+        if is_development() and progress is not None:
+            progress.close()

supervisely/convert/video/sly/sly_video_converter.py

@@ -12,6 +12,10 @@ from supervisely.video.video import validate_ext as validate_video_ext
 
 class SLYVideoConverter(VideoConverter):
 
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._supports_links = True
+
     def __str__(self) -> str:
         return AvailableVideoConverters.SLY
 
@@ -45,6 +49,8 @@ class SLYVideoConverter(VideoConverter):
             return False
 
     def validate_format(self) -> bool:
+        if self.upload_as_links and self._supports_links:
+            self._download_remote_ann_files()
         detected_ann_cnt = 0
         videos_list, ann_dict = [], {}
         for root, _, files in os.walk(self._input_data):
@@ -103,6 +109,8 @@ class SLYVideoConverter(VideoConverter):
             meta = self._meta
 
         if item.ann_data is None:
+            if self._upload_as_links:
+                return None
             return item.create_empty_annotation()
 
         try:
@@ -113,5 +121,5 @@ class SLYVideoConverter(VideoConverter):
                 ann_json = sly_video_helper.rename_in_json(ann_json, renamed_classes, renamed_tags)
             return VideoAnnotation.from_json(ann_json, meta)
         except Exception as e:
-            logger.warn(f"Failed to convert annotation: {repr(e)}")
+            logger.warning(f"Failed to convert annotation: {repr(e)}")
             return item.create_empty_annotation()

supervisely/convert/video/video_converter.py

@@ -1,6 +1,6 @@
 import os
 import subprocess
-from typing import Dict, Optional, Union
+from typing import Dict, Optional, Tuple, Union
 
 import cv2
 import magic
@@ -57,10 +57,22 @@ class VideoConverter(BaseConverter):
                 self._frame_count = frame_count
             self._custom_data = custom_data if custom_data is not None else {}
 
+        @property
+        def shape(self) -> Tuple[int, int]:
+            return self._shape
+
+        @shape.setter
+        def shape(self, shape: Optional[Tuple[int, int]] = None):
+            self._shape = shape if shape is not None else [None, None]
+
         @property
         def frame_count(self) -> int:
             return self._frame_count
 
+        @frame_count.setter
+        def frame_count(self, frame_count: int):
+            self._frame_count = frame_count
+
         @property
         def name(self) -> str:
             if self._name is not None:
@@ -75,11 +87,11 @@ class VideoConverter(BaseConverter):
             return VideoAnnotation(self._shape, self._frame_count)
 
     def __init__(
-            self,
-            input_data: str,
-            labeling_interface: Optional[Union[LabelingInterface, str]],
-            upload_as_links: bool,
-            remote_files_map: Optional[Dict[str, str]] = None,
+        self,
+        input_data: str,
+        labeling_interface: Optional[Union[LabelingInterface, str]],
+        upload_as_links: bool,
+        remote_files_map: Optional[Dict[str, str]] = None,
     ):
         super().__init__(input_data, labeling_interface, upload_as_links, remote_files_map)
         self._key_id_map: KeyIdMap = None
@@ -114,7 +126,9 @@ class VideoConverter(BaseConverter):
         existing_names = set([vid.name for vid in api.video.get_list(dataset_id)])
 
         # check video codecs, mimetypes and convert if needed
-        convert_progress, convert_progress_cb = self.get_progress(self.items_count, "Preparing videos...")
+        convert_progress, convert_progress_cb = self.get_progress(
+            self.items_count, "Preparing videos..."
+        )
         for item in self._items:
             item_name, item_path = self.convert_to_mp4_if_needed(item.path)
             item.name = item_name
@@ -124,15 +138,19 @@ class VideoConverter(BaseConverter):
             convert_progress.close()
 
         has_large_files = False
+        size_progress_cb = None
         progress_cb, progress, ann_progress, ann_progress_cb = None, None, None, None
-        if log_progress and not self.upload_as_links:
+        if log_progress:
             progress, progress_cb = self.get_progress(self.items_count, "Uploading videos...")
-            file_sizes = [get_file_size(item.path) for item in self._items]
-            has_large_files = any([self._check_video_file_size(file_size) for file_size in file_sizes])
-            if has_large_files:
-                upload_progress = []
-                size_progress_cb = self._get_video_upload_progress(upload_progress)
-        batch_size = 1 if has_large_files or not self.upload_as_links else batch_size
+            if not self.upload_as_links:
+                file_sizes = [get_file_size(item.path) for item in self._items]
+                has_large_files = any(
+                    [self._check_video_file_size(file_size) for file_size in file_sizes]
+                )
+                if has_large_files:
+                    upload_progress = []
+                    size_progress_cb = self._get_video_upload_progress(upload_progress)
+        batch_size = 1 if has_large_files and not self.upload_as_links else batch_size
 
         for batch in batched(self._items, batch_size=batch_size):
             item_names = []
@@ -146,17 +164,21 @@ class VideoConverter(BaseConverter):
                 item_paths.append(item.path)
                 item_names.append(item.name)
 
-                if not self.upload_as_links:
-                    # TODO: implement generating annotations for remote videos
+                ann = None
+                if not self.upload_as_links or self.supports_links:
                     ann = self.to_supervisely(item, meta, renamed_classes, renamed_tags)
-                    figures_cnt += len(ann.figures)
-                    anns.append(ann)
+                    if ann is not None:
+                        figures_cnt += len(ann.figures)
+                anns.append(ann)
 
             if self.upload_as_links:
                 vid_infos = api.video.upload_links(
                     dataset_id,
                     item_paths,
                     item_names,
+                    skip_download=True,
+                    progress_cb=progress_cb if log_progress else None,
+                    force_metadata_for_links=False,
                 )
             else:
                 vid_infos = api.video.upload_paths(
@@ -164,22 +186,24 @@ class VideoConverter(BaseConverter):
                     item_names,
                     item_paths,
                     progress_cb=progress_cb if log_progress else None,
-                    item_progress=size_progress_cb if log_progress and has_large_files else None, # pylint: disable=used-before-assignment
+                    item_progress=(size_progress_cb if log_progress and has_large_files else None),
                 )
-                vid_ids = [vid_info.id for vid_info in vid_infos]
+            vid_ids = [vid_info.id for vid_info in vid_infos]
 
-                if log_progress and has_large_files and figures_cnt > 0:
-                    ann_progress, ann_progress_cb = self.get_progress(figures_cnt, "Uploading annotations...")
+            if log_progress and has_large_files and figures_cnt > 0:
+                ann_progress, ann_progress_cb = self.get_progress(
+                    figures_cnt, "Uploading annotations..."
+                )
 
-                for video_id, ann in zip(vid_ids, anns):
-                    if ann is None:
-                        ann = VideoAnnotation(item.shape, item.frame_count)
-                    api.video.annotation.append(video_id, ann, progress_cb=ann_progress_cb)
+            for vid, ann, item, info in zip(vid_ids, anns, batch, vid_infos):
+                if ann is None:
+                    ann = VideoAnnotation((info.frame_height, info.frame_width), info.frames_count)
+                api.video.annotation.append(vid, ann, progress_cb=ann_progress_cb)
 
         if log_progress and is_development():
-            if progress is not None: # pylint: disable=possibly-used-before-assignment
+            if progress is not None:
                 progress.close()
-            if not self.upload_as_links and ann_progress is not None:
+            if ann_progress is not None:
                 ann_progress.close()
         logger.info(f"Dataset ID:{dataset_id} has been successfully uploaded.")
 
@@ -268,7 +292,7 @@ class VideoConverter(BaseConverter):
         )
 
     def _check_video_file_size(self, file_size):
-        return file_size > 20 * 1024 * 1024 # 20 MB
+        return file_size > 20 * 1024 * 1024  # 20 MB
 
     def _get_video_upload_progress(self, upload_progress):
         upload_progress = []