supervisely 6.73.342__py3-none-any.whl → 6.73.344__py3-none-any.whl

supervisely/api/dataset_api.py

@@ -4,7 +4,10 @@
 # docs
 from __future__ import annotations
 
+import os
+from pathlib import Path
 from typing import (
+    TYPE_CHECKING,
     Any,
     Dict,
     Generator,
@@ -16,7 +19,15 @@ from typing import (
     Union,
 )
 
-from supervisely._utils import abs_url, compress_image_url, is_development
+from tqdm import tqdm
+
+from supervisely._utils import (
+    abs_url,
+    compress_image_url,
+    is_development,
+    run_coroutine,
+)
+from supervisely.annotation.annotation import Annotation
 from supervisely.api.module_api import (
     ApiField,
     ModuleApi,
@@ -24,8 +35,14 @@ from supervisely.api.module_api import (
     UpdateableModule,
     _get_single_item,
 )
+from supervisely.io.json import load_json_file
 from supervisely.project.project_type import ProjectType
 
+if TYPE_CHECKING:
+    from supervisely.project.project import ProjectMeta
+
+from supervisely import logger
+
 
 class DatasetInfo(NamedTuple):
     """Represent Supervisely Dataset information.
@@ -1090,3 +1107,162 @@ class DatasetApi(UpdateableModule, RemoveableModuleApi):
         :rtype: bool
         """
         return self.get_info_by_name(project_id, name, parent_id=parent_id) is not None
+
+    def quick_import(
+        self,
+        dataset: Union[int, DatasetInfo],
+        blob_path: str,
+        offsets_path: str,
+        anns: List[str],
+        project_meta: Optional[ProjectMeta] = None,
+        project_type: Optional[ProjectType] = None,
+        log_progress: bool = True,
+    ):
+        """
+        Quick import of images and annotations to the dataset.
+        Used only for extended Supervisely format with blobs.
+        Project will be automatically marked as blob project.
+
+        IMPORTANT: Number of annotations must be equal to the number of images in offset file.
+                   Image names in the offset file and annotation files must match.
+
+        :param dataset: Dataset ID or DatasetInfo object.
+        :type dataset: Union[int, DatasetInfo]
+        :param blob_path: Local path to the blob file.
+        :type blob_path: str
+        :param offsets_path: Local path to the offsets file.
+        :type offsets_path: str
+        :param anns: List of annotation paths.
+        :type anns: List[str]
+        :param project_meta: ProjectMeta object.
+        :type project_meta: Optional[ProjectMeta], optional
+        :param project_type: Project type.
+        :type project_type: Optional[ProjectType], optional
+        :param log_progress: If True, show progress bar.
+        :type log_progress: bool, optional
+
+
+        :Usage example:
+
+        .. code-block:: python
+
+            import supervisely as sly
+            from supervisely.project.project_meta import ProjectMeta
+            from supervisely.project.project_type import ProjectType
+
+            api = sly.Api.from_env()
+
+            dataset_id = 123
+            workspace_id = 456
+            blob_path = "/path/to/blob"
+            offsets_path = "/path/to/offsets"
+            project_meta_path = "/path/to/project_meta.json"
+            anns = ["/path/to/ann1.json", "/path/to/ann2.json", ...]
+
+            # Create a new project, dataset and update its meta
+            project = api.project.create(
+                workspace_id,
+                "Quick Import",
+                type=sly.ProjectType.IMAGES,
+                change_name_if_conflict=True,
+            )
+            dataset = api.dataset.create(project.id, "ds1")
+            project_meta_json = sly.json.load_json_file(project_meta_path)
+            meta = api.project.update_meta(project.id, meta=project_meta_json)
+
+            dataset_info = api.dataset.quick_import(
+                dataset=dataset.id,
+                blob_path=blob_path,
+                offsets_path=offsets_path,
+                anns=anns,
+                project_meta=ProjectMeta(),
+                project_type=ProjectType.IMAGES,
+                log_progress=True
+            )
+
+        """
+        from supervisely.api.api import Api, ApiContext
+        from supervisely.api.image_api import _BLOB_TAG_NAME
+        from supervisely.project.project import TF_BLOB_DIR, ProjectMeta
+
+        def _ann_objects_generator(ann_paths, project_meta):
+            for ann in ann_paths:
+                ann_json = load_json_file(ann)
+                yield Annotation.from_json(ann_json, project_meta)
+
+        self._api: Api
+
+        if isinstance(dataset, int):
+            dataset = self.get_info_by_id(dataset)
+
+        project_info = self._api.project.get_info_by_id(dataset.project_id)
+
+        if project_meta is None:
+            meta_dict = self._api.project.get_meta(dataset.project_id)
+            project_meta = ProjectMeta.from_json(meta_dict)
+
+        if project_type is None:
+            project_type = project_info.type
+
+        if project_type != ProjectType.IMAGES:
+            raise NotImplementedError(
+                f"Quick import is not implemented for project type {project_type}"
+            )
+
+        # Set optimization context
+        with ApiContext(
+            api=self._api,
+            project_id=dataset.project_id,
+            dataset_id=dataset.id,
+            project_meta=project_meta,
+        ):
+            dst_blob_path = os.path.join(f"/{TF_BLOB_DIR}", os.path.basename(blob_path))
+            dst_offset_path = os.path.join(f"/{TF_BLOB_DIR}", os.path.basename(offsets_path))
+            if log_progress:
+                sizeb = os.path.getsize(blob_path) + os.path.getsize(offsets_path)
+                b_progress_cb = tqdm(
+                    total=sizeb,
+                    unit="B",
+                    unit_scale=True,
+                    desc=f"Uploading blob to file storage",
+                )
+            else:
+                b_progress_cb = None
+
+            self._api.file.upload_bulk_fast(
+                team_id=project_info.team_id,
+                src_paths=[blob_path, offsets_path],
+                dst_paths=[dst_blob_path, dst_offset_path],
+                progress_cb=b_progress_cb.update,
+            )
+
+            blob_file_id = self._api.file.get_info_by_path(project_info.team_id, dst_blob_path).id
+
+            if log_progress:
+                of_progress_cb = tqdm(desc=f"Uploading images by offsets", total=len(anns)).update
+            else:
+                of_progress_cb = None
+
+            image_info_generator = self._api.image.upload_by_offsets_generator(
+                dataset=dataset,
+                team_file_id=blob_file_id,
+                progress_cb=of_progress_cb,
+            )
+
+            ann_map = {Path(ann).stem: ann for ann in anns}
+
+            for image_info_batch in image_info_generator:
+                img_ids = [img_info.id for img_info in image_info_batch]
+                img_names = [img_info.name for img_info in image_info_batch]
+                img_anns = [ann_map[img_name] for img_name in img_names]
+                ann_objects = _ann_objects_generator(img_anns, project_meta)
+                coroutine = self._api.annotation.upload_anns_async(
+                    image_ids=img_ids, anns=ann_objects, log_progress=log_progress
+                )
+                run_coroutine(coroutine)
+        try:
+            custom_data = self._api.project.get_custom_data(dataset.project_id)
+            custom_data[_BLOB_TAG_NAME] = True
+            self._api.project.update_custom_data(dataset.project_id, custom_data)
+        except:
+            logger.warning("Failed to set blob tag for project")

supervisely/api/entity_annotation/figure_api.py

@@ -700,3 +700,87 @@ class FigureApi(RemoveableBulkModuleApi):
             raise RuntimeError("Not all geometries were downloaded")
         ordered_results = [geometries[i] for i in ids]
         return ordered_results
+
+    async def upload_geometries_batch_async(
+        self,
+        figure_ids: List[int],
+        geometries: List[dict],
+        semaphore: Optional[asyncio.Semaphore] = None,
+        progress_cb: Optional[Union[tqdm, Callable]] = None,
+    ) -> None:
+        """
+        Upload figure geometries with given figure IDs to storage asynchronously in batches.
+
+        :param figure_ids: List of figure IDs in Supervisely.
+        :type figure_ids: List[int]
+        :param geometries: List of figure geometries in Supervisely JSON format.
+        :type geometries: List[dict]
+        :param semaphore: Semaphore to limit the number of concurrent uploads.
+        :type semaphore: Optional[asyncio.Semaphore], optional
+        :param progress_cb: Progress bar to show the upload progress. Shows the number of geometries uploaded.
+        :type progress_cb: Union[tqdm, Callable], optional
+        :return: None
+        :rtype: None
+
+        :Usage example:
+
+            .. code-block:: python
+
+                import asyncio
+                import supervisely as sly
+
+                os.environ['SERVER_ADDRESS'] = 'https://app.supervisely.com'
+                os.environ['API_TOKEN'] = 'Your Supervisely API Token'
+                api = sly.Api.from_env()
+
+                figure_ids = [642155547, 642155548, 642155549]
+                geometries = [{...}, {...}, {...}]  # Your geometry data
+
+                upload_coroutine = api.figure.upload_geometries_batch_async(
+                        figure_ids,
+                        geometries,
+                        semaphore=asyncio.Semaphore(10),
+                    )
+                sly.run_coroutine(upload_coroutine)
+        """
+        if semaphore is None:
+            semaphore = self._api.get_default_semaphore()
+
+        encoded_geometries = [json.dumps(geometry).encode("utf-8") for geometry in geometries]
+
+        batch_size = 100
+        tasks = []
+
+        for batch_ids, batch_geometries in zip(
+            batched(figure_ids, batch_size),
+            batched(encoded_geometries, batch_size),
+        ):
+            fields = []
+            for figure_id, geometry in zip(batch_ids, batch_geometries):
+                fields.append((ApiField.FIGURE_ID, str(figure_id)))
+                fields.append(
+                    (
+                        ApiField.GEOMETRY,
+                        (str(figure_id), geometry, "application/octet-stream"),
+                    )
+                )
+
+            async def upload_batch(fields, progress_cb, num):
+                async with semaphore:
+                    encoder = MultipartEncoder(fields=fields)
+                    headers = {"Content-Type": encoder.content_type}
+                    async for _, _ in self._api.stream_async(
+                        "figures.bulk.upload.geometry",
+                        "POST",
+                        data=encoder,
+                        content=encoder.to_string(),
+                        headers=headers,
+                    ):
+                        pass
+                if progress_cb is not None:
+                    progress_cb.update(num)
+
+            tasks.append(upload_batch(fields, progress_cb, len(batch_ids)))
+
+        if tasks:
+            await asyncio.gather(*tasks)

supervisely/api/file_api.py

@@ -813,7 +813,7 @@ class FileApi(ModuleApiBase):
         :type src: List[str]
         :param dst: Destination paths for Files to Team Files.
         :type dst: List[str]
-        :param progress_cb: Function for tracking download progress.
+        :param progress_cb: Function for tracking upload progress.
         :type progress_cb: tqdm or callable, optional
         :return: Information about Files. See :class:`info_sequence<info_sequence>`
         :rtype: :class:`List[FileInfo]`
@@ -1546,7 +1546,7 @@ class FileApi(ModuleApiBase):
                 sly_fs.remove_dir(temp_path)
             return content
         else:
-            raise FileNotFoundError(f"File not found: {remote_path}")
+            raise FileNotFoundError(f"File not found in Team Files at path: {remote_path}")
 
     async def _download_async(
         self,