supervisely 6.73.222__tar.gz → 6.73.224__tar.gz

{supervisely-6.73.222/supervisely.egg-info → supervisely-6.73.224}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.222
+Version: 6.73.224
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/api/file_api.py

@@ -421,16 +421,17 @@ class FileApi(ModuleApiBase):
         remote_path,
         local_save_path,
         progress_cb=None,
-        show_progress: bool = False,
+        log_progress: bool = False,
     ):
         response = self._api.post(
             "file-storage.download",
             {ApiField.TEAM_ID: team_id, ApiField.PATH: remote_path},
             stream=True,
         )
-        if show_progress is False:
-            progress_cb = None
-        elif show_progress and progress_cb is None:
+        if progress_cb is not None:
+            log_progress = False
+
+        if log_progress and progress_cb is None:
             total_size = int(response.headers.get("Content-Length", 0))
             progress_cb = tqdm_sly(
                 total=total_size,

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/api/image_api.py

@@ -605,9 +605,13 @@ class ImageApi(RemoveableBulkModuleApi):
         infos_dict = {}
         ids_set = set(ids)
         while any(ids_set):
-            dataset_id = self.get_info_by_id(
-                ids_set.pop(), force_metadata_for_links=False
-            ).dataset_id
+            img_id = ids_set.pop()
+            image_info = self.get_info_by_id(img_id, force_metadata_for_links=False)
+            if image_info is None:
+                raise KeyError(
+                    f"Image (id: {img_id}) is either archived, doesn't exist or you don't have enough permissions to access it"
+                )
+            dataset_id = image_info.dataset_id
             for batch in batched(ids):
                 filters = [{"field": ApiField.ID, "operator": "in", "value": batch}]
                 temp_results = self.get_list_all_pages(
@@ -1755,12 +1759,13 @@ class ImageApi(RemoveableBulkModuleApi):
             raise ValueError(
                 f"Conflict resolution should be one of the following: {SUPPORTED_CONFLICT_RESOLUTIONS}"
             )
+        if len(set(names)) != len(names):
+            raise ValueError("Some image names are duplicated, only unique images can be uploaded.")
+
         results = []
 
         def _add_timestamp(name: str) -> str:
-
             now = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
-
             return f"{get_file_name(name)}_{now}{get_file_ext(name)}"
 
         def _pack_for_request(names: List[str], items: List[Any], metas: List[Dict]) -> List[Any]:
@@ -1816,6 +1821,8 @@ class ImageApi(RemoveableBulkModuleApi):
                     break
                 except HTTPError as e:
                     error_details = e.response.json().get("details", {})
+                    if isinstance(error_details, list):
+                        error_details = error_details[0]
                     if (
                         conflict_resolution is not None
                         and e.response.status_code == 400

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/api/video/video_api.py

@@ -413,7 +413,12 @@ class VideoApi(RemoveableBulkModuleApi):
             return_first_response=False,
         )
 
-    def get_info_by_id(self, id: int, raise_error: Optional[bool] = False) -> VideoInfo:
+    def get_info_by_id(
+        self,
+        id: int,
+        raise_error: Optional[bool] = False,
+        force_metadata_for_links=True,
+    ) -> VideoInfo:
         """
         Get Video information by ID in VideoInfo<VideoInfo> format.
 
@@ -421,6 +426,8 @@ class VideoApi(RemoveableBulkModuleApi):
         :type id: int
         :param raise_error: Return an error if the video info was not received.
         :type raise_error: bool
+        :param force_metadata_for_links: Get video metadata from server (if the video is uploaded as a link)
+        :type force_metadata_for_links: bool
         :return: Information about Video. See :class:`info_sequence<info_sequence>`
         :rtype: :class:`VideoInfo`
 
@@ -477,7 +484,11 @@ class VideoApi(RemoveableBulkModuleApi):
             # )
         """
 
-        info = self._get_info_by_id(id, "videos.info")
+        info = self._get_info_by_id(
+            id,
+            "videos.info",
+            fields={ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links},
+        )
         if info is None and raise_error is True:
             raise KeyError(f"Video with id={id} not found in your account")
         return info
@@ -546,6 +557,7 @@ class VideoApi(RemoveableBulkModuleApi):
                         ApiField.DATASET_ID: dataset_id,
                         ApiField.FILTER: filters,
                         ApiField.FIELDS: fields,
+                        ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
                     },
                 )
             )
@@ -555,7 +567,12 @@ class VideoApi(RemoveableBulkModuleApi):
         ordered_results = [temp_map[id] for id in ids]
         return ordered_results
 
-    def get_json_info_by_id(self, id: int, raise_error: Optional[bool] = False) -> Dict:
+    def get_json_info_by_id(
+        self,
+        id: int,
+        raise_error: Optional[bool] = False,
+        force_metadata_for_links: Optional[bool] = True,
+    ) -> Dict:
         """
         Get Video information by ID in json format.
 
@@ -629,7 +646,12 @@ class VideoApi(RemoveableBulkModuleApi):
         """
 
         data = None
-        response = self._get_response_by_id(id, "videos.info", id_field=ApiField.ID)
+        response = self._get_response_by_id(
+            id,
+            "videos.info",
+            id_field=ApiField.ID,
+            fields={ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links},
+        )
         if response is None:
             if raise_error is True:
                 raise KeyError(f"Video with id={id} not found in your account")
@@ -1050,7 +1072,14 @@ class VideoApi(RemoveableBulkModuleApi):
         return new_videos
 
     def _upload_bulk_add(
-        self, func_item_to_kv, dataset_id, names, items, metas=None, progress_cb=None
+        self,
+        func_item_to_kv,
+        dataset_id,
+        names,
+        items,
+        metas=None,
+        progress_cb=None,
+        force_metadata_for_links=True,
     ):
         if metas is None:
             metas = [{}] * len(items)
@@ -1077,7 +1106,11 @@ class VideoApi(RemoveableBulkModuleApi):
                 )
             response = self._api.post(
                 "videos.bulk.add",
-                {ApiField.DATASET_ID: dataset_id, ApiField.VIDEOS: images},
+                {
+                    ApiField.DATASET_ID: dataset_id,
+                    ApiField.VIDEOS: images,
+                    ApiField.FORCE_METADATA_FOR_LINKS: force_metadata_for_links,
+                },
             )
             if progress_cb is not None:
                 progress_cb(len(images))
@@ -1791,6 +1824,8 @@ class VideoApi(RemoveableBulkModuleApi):
         hashes: List[str] = None,
         metas: Optional[List[Dict]] = None,
         skip_download: Optional[bool] = False,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+        force_metadata_for_links: Optional[bool] = True,
     ) -> List[VideoInfo]:
         """
         Upload Videos from given links to Dataset.
@@ -1809,6 +1844,10 @@ class VideoApi(RemoveableBulkModuleApi):
         :type metas: List[dict], optional
         :param skip_download: Skip download videos to local storage.
         :type skip_download: Optional[bool]
+        :param progress_cb: Function for tracking the progress of copying.
+        :type progress_cb: tqdm or callable, optional
+        :param force_metadata_for_links: Specify if metadata should be forced. Default is True.
+        :type force_metadata_for_links: Optional[bool]
         :return: List with information about Videos. See :class:`info_sequence<info_sequence>`
         :rtype: :class:`List[VideoInfo]`
         :Usage example:
@@ -1843,7 +1882,13 @@ class VideoApi(RemoveableBulkModuleApi):
         # if infos is not None and hashes is not None and not skip_download:
         #     self.upsert_infos(hashes, infos, links)
         return self._upload_bulk_add(
-            lambda item: (ApiField.LINK, item), dataset_id, names, links, metas
+            lambda item: (ApiField.LINK, item),
+            dataset_id,
+            names,
+            links,
+            metas,
+            progress_cb=progress_cb,
+            force_metadata_for_links=force_metadata_for_links,
         )
 
     def update_custom_data(self, id: int, data: dict):

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/convert/base_converter.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import os
+from pathlib import Path
 from typing import Dict, List, Optional, Tuple, Union
 
 from tqdm import tqdm
@@ -10,7 +11,14 @@ from supervisely.annotation.annotation import Annotation
 from supervisely.annotation.tag_meta import TagValueType
 from supervisely.api.api import Api
 from supervisely.io.env import team_id
-from supervisely.io.fs import get_file_ext, get_file_name_with_ext, silent_remove
+from supervisely.io.fs import (
+    get_file_ext,
+    get_file_name_with_ext,
+    is_archive,
+    remove_dir,
+    silent_remove,
+    unpack_archive,
+)
 from supervisely.project.project_meta import ProjectMeta
 from supervisely.project.project_settings import LabelingInterface
 from supervisely.sly_logger import logger
@@ -433,30 +441,72 @@ class BaseConverter:
         if not self.upload_as_links:
             return
 
-        files_to_download = {
+        ann_archives = {l: r for l, r in self._remote_files_map.items() if is_archive(l)}
+
+        anns_to_download = {
             l: r for l, r in self._remote_files_map.items() if get_file_ext(l) == self.ann_ext
         }
-        if not files_to_download:
+        if not anns_to_download and not ann_archives:
             return
 
         import asyncio
 
-        loop = asyncio.get_event_loop()
-        _, progress_cb = self.get_progress(
-            len(files_to_download),
-            "Downloading annotation files from remote storage",
-        )
+        for files_type, files in {
+            "annotations": anns_to_download,
+            "archives": ann_archives,
+        }.items():
+            if not files:
+                continue
 
-        for local_path in files_to_download.keys():
-            silent_remove(local_path)
+            is_archive_type = files_type == "archives"
 
-        logger.info("Downloading annotation files from remote storage...")
-        loop.run_until_complete(
-            self._api.storage.download_bulk_async(
-                team_id=self._team_id,
-                remote_paths=list(files_to_download.values()),
-                local_save_paths=list(files_to_download.keys()),
-                progress_cb=progress_cb,
+            file_size = None
+            if is_archive_type:
+                logger.info(f"Remote archives detected.")
+                file_size = sum(
+                    self._api.storage.get_info_by_path(self._team_id, remote_path).sizeb
+                    for remote_path in files.values()
+                )
+
+            loop = asyncio.get_event_loop()
+            _, progress_cb = self.get_progress(
+                len(files) if not is_archive_type else file_size,
+                f"Downloading {files_type} from remote storage",
+                is_size=is_archive_type,
             )
-        )
-        logger.info("Annotation files downloaded successfully")
+
+            for local_path in files.keys():
+                silent_remove(local_path)
+
+            logger.info(f"Downloading {files_type} from remote storage...")
+            loop.run_until_complete(
+                self._api.storage.download_bulk_async(
+                    team_id=self._team_id,
+                    remote_paths=list(files.values()),
+                    local_save_paths=list(files.keys()),
+                    progress_cb=progress_cb,
+                    progress_cb_type="number" if not is_archive_type else "size",
+                )
+            )
+            logger.info("Possible annotations downloaded successfully.")
+
+            if is_archive_type:
+                for local_path in files.keys():
+                    parent_dir = Path(local_path).parent
+                    if parent_dir.name == "ann":
+                        target_dir = parent_dir
+                    else:
+                        target_dir = parent_dir / "ann"
+                        target_dir.mkdir(parents=True, exist_ok=True)
+
+                    unpack_archive(local_path, str(target_dir))
+                    silent_remove(local_path)
+
+                    dirs = [d for d in target_dir.iterdir() if d.is_dir()]
+                    files = [f for f in target_dir.iterdir() if f.is_file()]
+                    if len(dirs) == 1 and len(files) == 0:
+                        for file in dirs[0].iterdir():
+                            file.rename(target_dir / file.name)
+                        remove_dir(str(dirs[0]))
+
+                    logger.info(f"Archive {local_path} unpacked successfully to {str(target_dir)}")

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/convert/converter.py

@@ -169,21 +169,26 @@ class ImportManager:
         mkdir(local_path, remove_content_if_exists=True)
 
         if is_dir:
-            files = self._api.storage.list(self._team_id, remote_path)
+            files = self._api.storage.list(self._team_id, remote_path, include_folders=False)
         else:
             files = [self._api.storage.get_info_by_path(self._team_id, remote_path)]
 
+        unique_directories = set()
         for file in files:
             new_path = file.path.replace(dir_path, local_path)
             self._remote_files_map[new_path] = file.path
             Path(new_path).parent.mkdir(parents=True, exist_ok=True)
+            unique_directories.add(str(Path(file.path).parent))
             touch(new_path)
 
+        logger.info(f"Scanned remote directories:\n   - " + "\n   - ".join(unique_directories))
         return local_path
 
     def _unpack_archives(self, local_path):
         """Unpack if input data contains an archive."""
 
+        if self._upload_as_links:
+            return
         new_paths_to_scan = [local_path]
         while len(new_paths_to_scan) > 0:
             archives = []

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/convert/image/image_converter.py

@@ -114,6 +114,7 @@ class ImageConverter(BaseConverter):
         batch_size: int = 50,
         log_progress=True,
         entities: List[Item] = None,
+        progress_cb=None,
     ) -> None:
         """Upload converted data to Supervisely"""
         dataset_info = api.dataset.get_info_by_id(dataset_id, raise_error=True)
@@ -122,10 +123,14 @@ class ImageConverter(BaseConverter):
         meta, renamed_classes, renamed_tags = self.merge_metas_with_conflicts(api, dataset_id)
 
         existing_names = set([img.name for img in api.image.get_list(dataset_id)])
-        if log_progress:
-            progress, progress_cb = self.get_progress(self.items_count, "Uploading images...")
-        else:
-            progress_cb = None
+        progress = None
+        if progress_cb is not None:
+            log_progress = True
+        elif log_progress:
+            progress, progress_cb = self.get_progress(self.items_count, "Uploading")
+
+        if self.upload_as_links:
+            batch_size = 1000
 
         for batch in batched(entities or self._items, batch_size=batch_size):
             item_names = []
@@ -166,6 +171,7 @@ class ImageConverter(BaseConverter):
                         item_names,
                         item_paths,
                         metas=item_metas,
+                        batch_size=batch_size,
                         conflict_resolution="rename",
                         force_metadata_for_links=False,
                     )
@@ -188,9 +194,11 @@ class ImageConverter(BaseConverter):
                 progress_cb(len(batch))
 
         if log_progress:
-            if is_development():
+            if is_development() and progress is not None:
                 progress.close()
-        logger.info(f"Dataset ID:'{dataset_id}' has been successfully uploaded.")
+        logger.info(
+            f"Dataset has been successfully uploaded → {dataset_info.name}, ID:{dataset_id}"
+        )
 
     def validate_image(self, path: str) -> Tuple[str, str]:
         if self.upload_as_links:

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/convert/image/sly/sly_image_converter.py

@@ -12,7 +12,7 @@ from supervisely import (
     Rectangle,
     logger,
 )
-from supervisely._utils import generate_free_name
+from supervisely._utils import generate_free_name, is_development
 from supervisely.api.api import Api
 from supervisely.convert.base_converter import AvailableImageConverters
 from supervisely.convert.image.image_converter import ImageConverter
@@ -169,6 +169,8 @@ class SLYImageConverter(ImageConverter):
             project_dirs = [d for d in find_project_dirs(input_data)]
             if len(project_dirs) > 1:
                 logger.info("Found multiple possible Supervisely projects in the input data")
+            else:
+                logger.info("Possible Supervisely project found in the input data")
             meta = None
             for project_dir in project_dirs:
                 project_fs = Project(project_dir, mode=OpenMode.READ)
@@ -281,6 +283,12 @@ class SLYImageConverter(ImageConverter):
         existing_datasets = api.dataset.get_list(project_id, recursive=True)
         existing_datasets = {ds.name for ds in existing_datasets}
 
+        if log_progress:
+            progress, progress_cb = self.get_progress(self.items_count, "Uploading project")
+        else:
+            progress, progress_cb = None, None
+
+        logger.info("Uploading project structure")
         def _upload_project(
             project_structure: Dict,
             project_id: int,
@@ -288,7 +296,6 @@ class SLYImageConverter(ImageConverter):
             parent_id: Optional[int] = None,
             first_dataset=False,
         ):
-
             for ds_name, value in project_structure.items():
                 ds_name = generate_free_name(existing_datasets, ds_name, extend_used_names=True)
                 if first_dataset:
@@ -298,13 +305,17 @@ class SLYImageConverter(ImageConverter):
                     dataset_id = api.dataset.create(project_id, ds_name, parent_id=parent_id).id
 
                 items = value.get(DATASET_ITEMS, [])
+                nested_datasets = value.get(NESTED_DATASETS, {})
+                logger.info(f"Dataset: {ds_name}, items: {len(items)}, nested datasets: {len(nested_datasets)}")
                 if items:
                     super(SLYImageConverter, self).upload_dataset(
-                        api, dataset_id, batch_size, log_progress, entities=items
+                        api, dataset_id, batch_size, entities=items, progress_cb=progress_cb
                     )
 
-                nested_datasets = value.get(NESTED_DATASETS, {})
                 if nested_datasets:
                     _upload_project(nested_datasets, project_id, dataset_id, dataset_id)
 
         _upload_project(self._project_structure, project_id, dataset_id, first_dataset=True)
+
+        if is_development() and progress is not None:
+            progress.close()

{supervisely-6.73.222 → supervisely-6.73.224}/supervisely/convert/video/video_converter.py

@@ -140,16 +140,17 @@ class VideoConverter(BaseConverter):
         has_large_files = False
         size_progress_cb = None
         progress_cb, progress, ann_progress, ann_progress_cb = None, None, None, None
-        if log_progress and not self.upload_as_links:
+        if log_progress:
             progress, progress_cb = self.get_progress(self.items_count, "Uploading videos...")
-            file_sizes = [get_file_size(item.path) for item in self._items]
-            has_large_files = any(
-                [self._check_video_file_size(file_size) for file_size in file_sizes]
-            )
-            if has_large_files:
-                upload_progress = []
-                size_progress_cb = self._get_video_upload_progress(upload_progress)
-        batch_size = 1 if has_large_files or not self.upload_as_links else batch_size
+            if not self.upload_as_links:
+                file_sizes = [get_file_size(item.path) for item in self._items]
+                has_large_files = any(
+                    [self._check_video_file_size(file_size) for file_size in file_sizes]
+                )
+                if has_large_files:
+                    upload_progress = []
+                    size_progress_cb = self._get_video_upload_progress(upload_progress)
+        batch_size = 1 if has_large_files and not self.upload_as_links else batch_size
 
         for batch in batched(self._items, batch_size=batch_size):
             item_names = []
@@ -176,6 +177,8 @@ class VideoConverter(BaseConverter):
                     item_paths,
                     item_names,
                     skip_download=True,
+                    progress_cb=progress_cb if log_progress else None,
+                    force_metadata_for_links=False,
                 )
             else:
                 vid_infos = api.video.upload_paths(

{supervisely-6.73.222 → supervisely-6.73.224/supervisely.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: supervisely
-Version: 6.73.222
+Version: 6.73.224
 Summary: Supervisely Python SDK.
 Home-page: https://github.com/supervisely/supervisely
 Author: Supervisely