superannotate 4.4.30b3__tar.gz → 4.4.31.dev2__tar.gz

{superannotate-4.4.30b3/src/superannotate.egg-info → superannotate-4.4.31.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: superannotate
-Version: 4.4.30b3
+Version: 4.4.31.dev2
 Summary: Python SDK to SuperAnnotate platform
 Home-page: https://github.com/superannotateai/superannotate-python-sdk
 Author: SuperAnnotate AI

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/__init__.py

@@ -3,7 +3,7 @@ import os
 import sys
 
 
-__version__ = "4.4.30b3"
+__version__ = "4.4.31dev2"
 
 os.environ.update({"sa_version": __version__})
 sys.path.append(os.path.split(os.path.realpath(__file__))[0])

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/lib/app/interface/sdk_interface.py

@@ -505,20 +505,19 @@ class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta):
         ) -> Tuple[bool, typing.Any]:
             try:
                 for component in component_data:
-                    if (
-                        component["type"] == "webComponent"
-                        and component["id"] == component_pk
-                    ):
-                        return True, component.get("context")
-                    if (
-                        component["type"] in ("group", "grid")
-                        and "children" in component
-                    ):
+                    if "children" in component:
                         found, val = retrieve_context(
                             component["children"], component_pk
                         )
                         if found:
                             return found, val
+                    if (
+                        "id" in component
+                        and component["id"] == component_pk
+                        and component["type"] == "webComponent"
+                    ):
+                        return True, json.loads(component.get("context"))
+
             except KeyError as e:
                 logger.debug("Got key error:", component_data)
                 raise e
@@ -2920,32 +2919,80 @@ class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta):
         project: NotEmptyStr,
         integration: Union[NotEmptyStr, IntegrationEntity],
         folder_path: Optional[NotEmptyStr] = None,
+        *,
+        query: Optional[NotEmptyStr] = None,
+        item_name_column: Optional[NotEmptyStr] = None,
+        custom_item_name: Optional[NotEmptyStr] = None,
+        component_mapping: Optional[Dict[str, str]] = None,
     ):
-        """Link images from integrated external storage to SuperAnnotate.
+        """Link images from integrated external storage to SuperAnnotate from AWS, GCP, Azure, Databricks.
 
         :param project: project name or folder path where items should be attached (e.g., “project1/folder1”).
         :type project: str
 
-        :param integration:  existing integration name or metadata dict to pull items from.
-         Mandatory keys in integration metadata’s dict is “name”.
+        :param integration: The existing integration name or metadata dict to pull items from.
+            Mandatory keys in integration metadata’s dict is “name”.
         :type integration: str or dict
 
         :param folder_path: Points to an exact folder/directory within given storage.
-         If None, items     are fetched from the root directory.
+            If None, items     are fetched from the root directory.
         :type folder_path: str
+
+        :param query: (Only for Databricks). The SQL query to retrieve specific columns from Databricks.
+            If provided, the function will execute the query and use the results for mapping and uploading.
+        :type query: Optional[str]
+
+        :param item_name_column: (Only for Databricks). The column name from the SQL query whose values
+            will be used as item names. If this is provided, custom_item_name cannot be used.
+            The column must exist in the query result.
+        :type item_name_column: Optional[str]
+
+        :param custom_item_name: (Only for Databricks). A manually defined prefix for item names.
+            A random 10-character suffix will be appended to ensure uniqueness.
+            If this is provided, item_name_column cannot be used.
+        :type custom_item_name: Optional[str]
+
+        :param component_mapping: (Only for Databricks). A dictionary mapping Databricks
+            columns to SuperAnnotate component IDs.
+        :type component_mapping: Optional[dict]
+
+
+        Request Example:
+        ::
+
+            client.attach_items_from_integrated_storage(
+                project="project_name",
+                integration="databricks_integration",
+                query="SELECT * FROM integration_data LIMIT 10",
+                item_name_column="prompt",
+                component_mapping={
+                    "category": "_item_category",
+                    "prompt_id": "id",
+                    "prompt": "prompt"
+                }
+            )
+
         """
         project, folder = self.controller.get_project_folder_by_path(project)
         _integration = None
         if isinstance(integration, str):
             integration = IntegrationEntity(name=integration)
         for i in self.controller.integrations.list().data:
-            if integration.name == i.name:
+            if integration.name.lower() == i.name.lower():
                 _integration = i
                 break
         else:
             raise AppException("Integration not found.")
+
         response = self.controller.integrations.attach_items(
-            project, folder, _integration, folder_path
+            project=project,
+            folder=folder,
+            integration=_integration,
+            folder_path=folder_path,
+            query=query,
+            item_name_column=item_name_column,
+            custom_item_name=custom_item_name,
+            component_mapping=component_mapping,
         )
         if response.errors:
             raise AppException(response.errors)
@@ -3594,7 +3641,7 @@ class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta):
             "skip", "replace", "replace_annotations_only"
         ] = "skip",
     ):
-        """Copy images in bulk between folders in a project
+        """Copy items in bulk between folders in a project
 
         :param source: project name (root) or folder path to pick items from (e.g., “project1/folder1”).
         :type source: str
@@ -3658,7 +3705,7 @@ class SAClient(BaseInterfaceFacade, metaclass=TrackableMeta):
             "skip", "replace", "replace_annotations_only"
         ] = "skip",
     ):
-        """Move images in bulk between folders in a project
+        """Move items in bulk between folders in a project
 
         :param source: project name (root) or folder path to pick items from (e.g., “project1/folder1”).
         :type source: str

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/lib/core/serviceproviders.py

@@ -490,6 +490,7 @@ class BaseAnnotationService(SuperannotateServiceProvider):
         self,
         project: entities.ProjectEntity,
         item_ids: List[int],
+        chunk_size: int = 1000,
     ) -> Dict[str, List]:
         raise NotImplementedError
 
@@ -592,6 +593,7 @@ class BaseIntegrationService(SuperannotateServiceProvider):
         folder: entities.FolderEntity,
         integration: entities.IntegrationEntity,
         folder_name: str = None,
+        options: Dict[str, str] = None,
     ) -> ServiceResponse:
         raise NotImplementedError
 

superannotate-4.4.31.dev2/src/superannotate/lib/core/usecases/integrations.py

@@ -0,0 +1,179 @@
+from typing import Dict
+from typing import Optional
+
+from lib.core.conditions import Condition
+from lib.core.conditions import CONDITION_EQ as EQ
+from lib.core.entities import FolderEntity
+from lib.core.entities import IntegrationEntity
+from lib.core.entities import ProjectEntity
+from lib.core.entities.integrations import IntegrationTypeEnum
+from lib.core.enums import ProjectType
+from lib.core.exceptions import AppException
+from lib.core.reporter import Reporter
+from lib.core.response import Response
+from lib.core.serviceproviders import BaseServiceProvider
+from lib.core.usecases import BaseReportableUseCase
+
+
+class GetIntegrations(BaseReportableUseCase):
+    def __init__(self, reporter: Reporter, service_provider: BaseServiceProvider):
+
+        super().__init__(reporter)
+        self._service_provider = service_provider
+
+    def execute(self) -> Response:
+        integrations = self._service_provider.integrations.list().data.integrations
+        integrations = list(sorted(integrations, key=lambda x: x.createdAt))
+        integrations.reverse()
+        self._response.data = integrations
+        return self._response
+
+
+class AttachIntegrations(BaseReportableUseCase):
+    MULTIMODAL_INTEGRATIONS = [
+        IntegrationTypeEnum.DATABRICKS,
+        IntegrationTypeEnum.SNOWFLAKE,
+    ]
+
+    def __init__(
+        self,
+        reporter: Reporter,
+        project: ProjectEntity,
+        folder: FolderEntity,
+        service_provider: BaseServiceProvider,
+        integration: IntegrationEntity,
+        folder_path: str = None,
+        query: Optional[str] = None,
+        item_name_column: Optional[str] = None,
+        custom_item_name: Optional[str] = None,
+        component_mapping: Optional[Dict[str, str]] = None,
+    ):
+        super().__init__(reporter)
+        self._project = project
+        self._folder = folder
+        self._integration = integration
+        self._service_provider = service_provider
+        self._folder_path = folder_path
+        self._query = query
+        self._item_name_column = item_name_column
+        self._custom_item_name = custom_item_name
+        self._component_mapping = component_mapping
+        self._options = {}  # using only for Databricks and Snowflake
+        self._item_category_column = None
+
+    @property
+    def _upload_path(self):
+        return f"{self._project.name}{f'/{self._folder.name}' if self._folder.name != 'root' else ''}"
+
+    def validate_integration(self):
+        # TODO add support in next iterations
+        if self._integration.type == IntegrationTypeEnum.SNOWFLAKE:
+            raise AppException(
+                "Attaching items is not supported with Snowflake integration."
+            )
+
+        if self._integration.type in self.MULTIMODAL_INTEGRATIONS:
+            if self._project.type != ProjectType.MULTIMODAL:
+                raise AppException(
+                    f"{self._integration.name} integration is supported only for Multimodal projects."
+                )
+
+    def validate_options_for_multimodal_integration(self):
+        if self._integration.type in self.MULTIMODAL_INTEGRATIONS:
+            if self._item_name_column and self._custom_item_name:
+                raise AppException(
+                    "‘item_name_column and custom_item_name cannot be used simultaneously."
+                )
+
+            if not self._item_name_column and not self._custom_item_name:
+                raise AppException(
+                    "Either item_name_column or custom_item_name is required."
+                )
+
+            if not all((self._query, self._component_mapping)):
+                raise AppException(
+                    f"{self._integration.name} integration requires both a query and component_mapping."
+                )
+
+            category_setting: bool = bool(
+                next(
+                    (
+                        setting.value
+                        for setting in self._service_provider.projects.list_settings(
+                            self._project
+                        ).data
+                        if setting.attribute == "CategorizeItems"
+                    ),
+                    None,
+                )
+            )
+            if (
+                not category_setting
+                and "_item_category" in self._component_mapping.values()
+            ):
+                raise AppException(
+                    "Item Category must be enabled for a project to use _item_category"
+                )
+
+            self._item_category_column = next(
+                (
+                    k
+                    for k, v in self._component_mapping.items()
+                    if v == "_item_category"
+                ),
+                None,
+            )
+            if self._item_category_column:
+                del self._component_mapping[self._item_category_column]
+
+            sa_components = [
+                c.name.lower()
+                for c in self._service_provider.annotation_classes.list(
+                    condition=Condition("project_id", self._project.id, EQ)
+                ).data
+            ]
+
+            for i in self._component_mapping.values():
+                if i.lower() not in sa_components:
+                    raise AppException(
+                        f"Component mapping contains invalid component ID: `{i}`"
+                    )
+
+    def generate_options_for_multimodal_integration(self):
+        self._options["query"] = self._query
+        self._options["item_name"] = (
+            self._custom_item_name if self._custom_item_name else self._item_name_column
+        )
+        self._options["prefix"] = True if self._custom_item_name else False
+        self._options["column_class_map"] = self._component_mapping
+        if self._item_category_column:
+            self._options["item_category"] = self._item_category_column
+
+    def execute(self) -> Response:
+        if self.is_valid():
+            if self._integration.type in self.MULTIMODAL_INTEGRATIONS:
+                self.generate_options_for_multimodal_integration()
+
+            self.reporter.log_info(
+                "Attaching file(s) from "
+                f"{self._integration.root}{f'/{self._folder_path}' if self._folder_path else ''} "
+                f"to {self._upload_path}. This may take some time."
+            )
+
+            attache_response = self._service_provider.integrations.attach_items(
+                project=self._project,
+                folder=self._folder,
+                integration=self._integration,
+                folder_name=self._folder_path
+                if self._integration.type not in self.MULTIMODAL_INTEGRATIONS
+                else None,
+                options=self._options if self._options else None,
+            )
+            if not attache_response.ok:
+                self._response.errors = AppException(
+                    f"An error occurred for {self._integration.name}. Please make sure: "
+                    "\n - The bucket exists."
+                    "\n - The connection is valid."
+                    "\n - The path to a specified directory is correct."
+                )
+            return self._response

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/lib/infrastructure/controller.py

@@ -1138,7 +1138,11 @@ class IntegrationManager(BaseManager):
         project: ProjectEntity,
         folder: FolderEntity,
         integration: IntegrationEntity,
-        folder_path: str,
+        folder_path: str = None,
+        query: Optional[str] = None,
+        item_name_column: Optional[str] = None,
+        custom_item_name: Optional[str] = None,
+        component_mapping: Optional[Dict[str, str]] = None,
     ):
         use_case = usecases.AttachIntegrations(
             reporter=Reporter(),
@@ -1147,6 +1151,10 @@ class IntegrationManager(BaseManager):
             folder=folder,
             integration=integration,
             folder_path=folder_path,
+            query=query,
+            item_name_column=item_name_column,
+            custom_item_name=custom_item_name,
+            component_mapping=component_mapping,
         )
         return use_case.execute()
 

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/lib/infrastructure/services/annotation.py

@@ -18,6 +18,8 @@ from lib.core.service_types import UploadAnnotations
 from lib.core.service_types import UploadAnnotationsResponse
 from lib.core.serviceproviders import BaseAnnotationService
 from lib.infrastructure.stream_data_handler import StreamedAnnotations
+from lib.infrastructure.utils import annotation_is_valid
+from lib.infrastructure.utils import divide_to_chunks
 
 try:
     from pydantic.v1 import parse_obj_as
@@ -170,21 +172,29 @@ class AnnotationService(BaseAnnotationService):
         self,
         project: entities.ProjectEntity,
         item_ids: List[int],
+        chunk_size: int = 1000,
     ) -> Dict[str, List]:
-        response_data = {"small": [], "large": []}
-        response = self.client.request(
-            url=urljoin(self.get_assets_provider_url(), self.URL_CLASSIFY_ITEM_SIZE),
-            method="POST",
-            params={"limit": len(item_ids)},
-            data={"project_id": project.id, "item_ids": item_ids},
-        )
-        if not response.ok:
-            raise AppException(response.error)
-        response_data["small"] = [
-            i["data"] for i in response.data.get("small", {}).values()
-        ]
-        response_data["large"] = response.data.get("large", [])
-        return response_data
+        small = []
+        large = []
+
+        chunks = divide_to_chunks(item_ids, chunk_size)
+        for chunk in chunks:
+            response = self.client.request(
+                method="POST",
+                url=urljoin(
+                    self.get_assets_provider_url(), self.URL_CLASSIFY_ITEM_SIZE
+                ),
+                params={"limit": len(chunk)},
+                data={
+                    "project_id": project.id,
+                    "item_ids": chunk,
+                },
+            )
+            if not response.ok:
+                raise AppException(response.error)
+            small.extend([i["data"] for i in response.data.get("small", {}).values()])
+            large.extend(response.data.get("large", []))
+        return {"small": small, "large": large}
 
     async def download_big_annotation(
         self,
@@ -218,6 +228,14 @@ class AnnotationService(BaseAnnotationService):
         ) as session:
             start_response = await session.request("post", url, params=query_params)
             res = await start_response.json()
+            if start_response.status > 299 or not annotation_is_valid(res):
+                logger.debug(
+                    f"Failed to download large annotation; item_id [{item_id}];"
+                    f" response: {res}; http_status: {start_response.status}"
+                )
+                raise AppException(
+                    f"Failed to download large annotation, ID: {item_id}"
+                )
             Path(download_path).mkdir(exist_ok=True, parents=True)
 
             dest_path = Path(download_path) / (item_name + ".json")

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/lib/infrastructure/services/integration.py

@@ -1,3 +1,5 @@
+from typing import Dict
+
 from lib.core import entities
 from lib.core.service_types import IntegrationListResponse
 from lib.core.serviceproviders import BaseIntegrationService
@@ -23,6 +25,7 @@ class IntegrationService(BaseIntegrationService):
         folder: entities.FolderEntity,
         integration: entities.IntegrationEntity,
         folder_name: str = None,
+        options: Dict[str, str] = None,
     ):
         data = {
             "team_id": project.team_id,
@@ -32,6 +35,8 @@ class IntegrationService(BaseIntegrationService):
         }
         if folder_name:
             data["customer_folder_name"] = folder_name
+        if options:
+            data["options"] = options
         return self.client.request(
             self.URL_ATTACH_INTEGRATIONS.format(project.team_id), "post", data=data
         )

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/lib/infrastructure/stream_data_handler.py

@@ -6,8 +6,12 @@ import typing
 from typing import Callable
 
 import aiohttp
+from lib.core.exceptions import AppException
+from lib.core.exceptions import BackendError
 from lib.core.reporter import Reporter
 from lib.infrastructure.services.http_client import AIOHttpSession
+from lib.infrastructure.utils import annotation_is_valid
+from lib.infrastructure.utils import async_retry_on_generator
 
 _seconds = 2**10
 TIMEOUT = aiohttp.ClientTimeout(
@@ -42,6 +46,7 @@ class StreamedAnnotations:
             self._reporter.log_error(f"Invalud chunk: {str(e)}")
             return None
 
+    @async_retry_on_generator((BackendError,))
     async def fetch(
         self,
         method: str,
@@ -59,6 +64,7 @@ class StreamedAnnotations:
         buffer = ""
         line_groups = b""
         decoder = json.JSONDecoder()
+        data_received = False
         async for line in response.content.iter_any():
             line_groups += line
             try:
@@ -71,6 +77,19 @@ class StreamedAnnotations:
                     if buffer.startswith(self.DELIMITER):
                         buffer = buffer[self.DELIMITER_LEN :]
                     json_obj, index = decoder.raw_decode(buffer)
+                    if not annotation_is_valid(json_obj):
+                        logger.warning(
+                            f"Invalid JSON detected in small annotations stream process, json: {json_obj}."
+                        )
+                        if data_received:
+                            raise AppException(
+                                "Invalid JSON detected in small annotations stream process."
+                            )
+                        else:
+                            raise BackendError(
+                                "Invalid JSON detected at the start of the small annotations stream process."
+                            )
+                    data_received = True
                     yield json_obj
                     if len(buffer[index:]) >= self.DELIMITER_LEN:
                         buffer = buffer[index + self.DELIMITER_LEN :]

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2}/src/superannotate/lib/infrastructure/utils.py

@@ -1,12 +1,17 @@
+import asyncio
+import logging
 import time
 from abc import ABC
 from abc import abstractmethod
+from functools import wraps
 from itertools import islice
 from pathlib import Path
 from typing import Any
+from typing import Callable
 from typing import Dict
 from typing import Optional
 from typing import Tuple
+from typing import Type
 from typing import Union
 
 from lib.core.entities import ProjectEntity
@@ -16,6 +21,9 @@ from lib.core.exceptions import PathError
 from lib.infrastructure.services.work_management import WorkManagementService
 
 
+logger = logging.getLogger("sa")
+
+
 def divide_to_chunks(it, size):
     it = iter(it)
     return iter(lambda: tuple(islice(it, size)), ())
@@ -44,6 +52,66 @@ def extract_project_folder(user_input: Union[str, dict]) -> Tuple[str, Optional[
     raise PathError("Invalid project path")
 
 
+def async_retry_on_generator(
+    exceptions: Tuple[Type[Exception]],
+    retries: int = 3,
+    delay: float = 0.3,
+    backoff: float = 0.3,
+):
+    """
+    An async retry decorator that retries a function only on specific exceptions.
+
+    Parameters:
+        exceptions (tuple): Tuple of exception classes to retry on.
+        retries (int): Number of retry attempts.
+        delay (float): Initial delay between retries in seconds.
+        backoff (float): Factor to increase the delay after each failure.
+    """
+
+    def decorator(func: Callable):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            attempt = 0
+            current_delay = delay
+            raised_exception = None
+
+            while attempt < retries:
+                try:
+                    async for v in func(*args, **kwargs):
+                        yield v
+                    return
+                except exceptions as e:
+                    raised_exception = e
+                    logger.debug(
+                        f"Attempt {attempt + 1}/{retries} failed with error: {e}. "
+                        f"Retrying in {current_delay} seconds..."
+                    )
+                    await asyncio.sleep(current_delay)
+                    current_delay += backoff  # Exponential backoff
+                finally:
+                    attempt += 1
+            if raised_exception:
+                logger.error(
+                    f"All {retries} attempts failed due to {raised_exception}."
+                )
+                raise raised_exception
+
+        return wrapper
+
+    return decorator
+
+
+def annotation_is_valid(annotation: dict) -> bool:
+    annotation_keys = annotation.keys()
+    if (
+        "errors" in annotation_keys
+        or "error" in annotation_keys
+        or "metadata" not in annotation_keys
+    ):
+        return False
+    return True
+
+
 class BaseCachedWorkManagementRepository(ABC):
     def __init__(self, ttl_seconds: int, work_management: WorkManagementService):
         self.ttl_seconds = ttl_seconds
@@ -80,10 +148,12 @@ class RoleCache(BaseCachedWorkManagementRepository):
         roles = response.data["data"]
         self._K_V_map[project.id] = {
             "role_name_id_map": {
-                role["role"]["name"]: role["role_id"] for role in roles
+                **{role["role"]["name"]: role["role_id"] for role in roles},
+                "ProjectAdmin": 3
             },
             "role_id_name_map": {
-                role["role_id"]: role["role"]["name"] for role in roles
+                **{role["role_id"]: role["role"]["name"] for role in roles},
+                3: "ProjectAdmin"
             },
         }
         self._update_cache_timestamp(project.id)

{superannotate-4.4.30b3 → superannotate-4.4.31.dev2/src/superannotate.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: superannotate
-Version: 4.4.30b3
+Version: 4.4.31.dev2
 Summary: Python SDK to SuperAnnotate platform
 Home-page: https://github.com/superannotateai/superannotate-python-sdk
 Author: SuperAnnotate AI