synapse-sdk 1.0.0a32__tar.gz → 1.0.0a34__tar.gz

{synapse_sdk-1.0.0a32/synapse_sdk.egg-info → synapse_sdk-1.0.0a34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a32
+Version: 1.0.0a34
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT

{synapse_sdk-1.0.0a32 → synapse_sdk-1.0.0a34}/synapse_sdk/clients/backend/__init__.py

@@ -1,6 +1,7 @@
 from synapse_sdk.clients.backend.annotation import AnnotationClientMixin
 from synapse_sdk.clients.backend.core import CoreClientMixin
 from synapse_sdk.clients.backend.dataset import DatasetClientMixin
+from synapse_sdk.clients.backend.hitl import HITLClientMixin
 from synapse_sdk.clients.backend.integration import IntegrationClientMixin
 from synapse_sdk.clients.backend.ml import MLClientMixin
 
@@ -11,6 +12,7 @@ class BackendClient(
     DatasetClientMixin,
     IntegrationClientMixin,
     MLClientMixin,
+    HITLClientMixin,
 ):
     name = 'Backend'
     token = None

{synapse_sdk-1.0.0a32 → synapse_sdk-1.0.0a34}/synapse_sdk/clients/backend/annotation.py

@@ -11,14 +11,14 @@ class AnnotationClientMixin(BaseClient):
         path = f'task_tags/{pk}/'
         return self._get(path)
 
-    def list_task_tags(self, data):
+    def list_task_tags(self, params):
         path = 'task_tags/'
-        return self._list(path, data=data)
+        return self._list(path, params=params)
 
-    def list_tasks(self, data, url_conversion=None, list_all=False):
+    def list_tasks(self, params=None, url_conversion=None, list_all=False):
         path = 'tasks/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
-        return self._list(path, data=data, url_conversion=url_conversion, list_all=list_all)
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
 
     def create_tasks(self, data):
         path = 'tasks/'

synapse_sdk-1.0.0a34/synapse_sdk/clients/backend/hitl.py

@@ -0,0 +1,17 @@
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_default_url_conversion
+
+
+class HITLClientMixin(BaseClient):
+    def get_assignment(self, pk):
+        path = f'assignments/{pk}/'
+        return self._get(path)
+
+    def list_assignments(self, params=None, url_conversion=None, list_all=False):
+        path = 'assignments/'
+        url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
+        return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def set_tags_assignments(self, data, params=None):
+        path = 'assignments/set_tags/'
+        return self._post(path, payload=data, params=params)

{synapse_sdk-1.0.0a32 → synapse_sdk-1.0.0a34}/synapse_sdk/clients/backend/integration.py

@@ -1,3 +1,4 @@
+from synapse_sdk.clients.backend.models import Storage
 from synapse_sdk.clients.base import BaseClient
 from synapse_sdk.utils.file import convert_file_to_base64
 
@@ -79,5 +80,6 @@ class IntegrationClientMixin(BaseClient):
         return self._list(path, params=params, list_all=list_all)
 
     def get_storage(self, pk):
+        """Get specific storage data from synapse backend."""
         path = f'storages/{pk}/'
-        return self._get(path)
+        return self._get(path, pydantic_model=Storage)

synapse_sdk-1.0.0a34/synapse_sdk/clients/backend/models.py

@@ -0,0 +1,44 @@
+from enum import Enum
+from typing import Dict
+
+from pydantic import BaseModel
+
+
+class StorageCategory(str, Enum):
+    """Synapse Backend Storage Category Enum."""
+
+    INTERNAL = 'internal'
+    EXTERNAL = 'external'
+
+
+class StorageProvider(str, Enum):
+    """Synapse Backend Storage Provider Enum."""
+
+    AMAZON_S3 = 'amazon_s3'
+    AZURE = 'azure'
+    DIGITAL_OCEAN = 'digital_ocean'
+    FILE_SYSTEM = 'file_system'
+    FTP = 'ftp'
+    SFTP = 'sftp'
+    MINIO = 'minio'
+    GCP = 'gcp'
+
+
+class Storage(BaseModel):
+    """Synapse Backend Storage Model.
+
+    Attrs:
+        id (int): The storage pk.
+        name (str): The storage name.
+        category (str): The storage category. (ex: internal, external)
+        provider (str): The storage provider. (ex: s3, gcp)
+        configuration (Dict): The storage configuration.
+        is_default (bool): The storage is default for Synapse backend workspace.
+    """
+
+    id: int
+    name: str
+    category: StorageCategory
+    provider: StorageProvider
+    configuration: Dict
+    is_default: bool

{synapse_sdk-1.0.0a32 → synapse_sdk-1.0.0a34}/synapse_sdk/clients/base.py

@@ -85,26 +85,59 @@ class BaseClient:
         except ValueError:
             return response.text
 
-    def _get(self, path, url_conversion=None, **kwargs):
+    def _get(self, path, url_conversion=None, pydantic_model=None, **kwargs):
+        """
+        Perform a GET request and optionally convert response to a pydantic model.
+
+        Args:
+            path (str): URL path to request.
+            url_conversion (dict, optional): Configuration for URL to path conversion.
+            pydantic_model (Type, optional): Pydantic model to convert the response to.
+            **kwargs: Additional keyword arguments to pass to the request.
+
+        Returns:
+            The response data, optionally converted to a pydantic model.
+        """
         response = self._request('get', path, **kwargs)
+
         if url_conversion:
             if url_conversion['is_list']:
                 files_url_to_path_from_objs(response['results'], **url_conversion, is_async=True)
             else:
                 files_url_to_path_from_objs(response, **url_conversion)
+
+        if pydantic_model:
+            return self._validate_response_with_pydantic_model(response, pydantic_model)
+
         return response
 
-    def _post(self, path, **kwargs):
-        return self._request('post', path, **kwargs)
+    def _post(self, path, pydantic_model=None, **kwargs):
+        response = self._request('post', path, **kwargs)
+        if pydantic_model:
+            return self._validate_response_with_pydantic_model(response, pydantic_model)
+        else:
+            return response
 
-    def _put(self, path, **kwargs):
-        return self._request('put', path, **kwargs)
+    def _put(self, path, pydantic_model=None, **kwargs):
+        response = self._request('put', path, **kwargs)
+        if pydantic_model:
+            return self._validate_response_with_pydantic_model(response, pydantic_model)
+        else:
+            return response
 
-    def _patch(self, path, **kwargs):
-        return self._request('patch', path, **kwargs)
+    def _patch(self, path, pydantic_model=None, **kwargs):
+        response = self._request('patch', path, **kwargs)
+        if pydantic_model:
+            return self._validate_response_with_pydantic_model(response, pydantic_model)
+        else:
+            return response
 
-    def _delete(self, path, **kwargs):
-        return self._request('delete', path, **kwargs)
+    def _delete(self, path, pydantic_model=None, **kwargs):
+        response = self._request('delete', path, **kwargs)
+        if pydantic_model:
+            return self._validate_response_with_pydantic_model(response, pydantic_model)
+        else:
+            return response
 
     def _list(self, path, url_conversion=None, list_all=False, **kwargs):
         response = self._get(path, **kwargs)
@@ -121,3 +154,17 @@ class BaseClient:
 
     def exists(self, api, *args, **kwargs):
         return getattr(self, api)(*args, **kwargs)['count'] > 0
+
+    def _validate_response_with_pydantic_model(self, response, pydantic_model):
+        """Validate a response with a pydantic model."""
+        # Check if model is a pydantic model (has the __pydantic_model__ attribute)
+        if (
+            hasattr(pydantic_model, '__pydantic_model__')
+            or hasattr(pydantic_model, 'model_validate')
+            or hasattr(pydantic_model, 'parse_obj')
+        ):
+            pydantic_model.model_validate(response)
+            return response
+        else:
+            # Not a pydantic model
+            raise TypeError('The provided model is not a pydantic model')

synapse_sdk-1.0.0a34/synapse_sdk/clients/validators/collections.py

@@ -0,0 +1,31 @@
+class FileSpecificationValidator:
+    """File specification validator class for synapse backend collection.
+
+    Args:
+        file_spec_template (list):
+            * List of dictionaries containing file specification template
+            * This is from synapse-backend file specification data.
+        organized_files (list): List of dictionaries containing organized files.
+    """
+
+    def __init__(self, file_spec_template, organized_files):
+        self.file_spec_template = file_spec_template
+        self.organized_files = organized_files
+
+    def validate(self):
+        """Validate the file specification template with organized files.
+
+        Returns:
+            bool: True if the file specification template is valid, False otherwise.
+        """
+        for spec in self.file_spec_template:
+            spec_name = spec['name']
+            is_required = spec['is_required']
+
+            for file_group in self.organized_files:
+                files = file_group['files']
+                if is_required and spec_name not in files:
+                    return False
+                if spec_name in files and not files[spec_name]:
+                    return False
+        return True

synapse_sdk-1.0.0a34/synapse_sdk/plugins/categories/export/actions/export.py

@@ -0,0 +1,222 @@
+from abc import ABC, abstractmethod
+from typing import Any, Literal
+
+from pydantic import BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class ExportTargetHandler(ABC):
+    """
+    Abstract base class for handling export targets.
+
+    This class defines the blueprint for export target handlers, requiring the implementation
+    of methods to validate filters, retrieve results, and process collections of results.
+    """
+
+    @abstractmethod
+    def validate_filter(self, value: dict, client: Any):
+        """
+        Validate filter query params to request original data from api.
+
+        Args:
+            value (dict): The filter criteria to validate.
+            client (Any): The client used to validate the filter.
+
+        Raises:
+            PydanticCustomError: If the filter criteria are invalid.
+
+        Returns:
+            dict: The validated filter criteria.
+        """
+        pass
+
+    @abstractmethod
+    def get_results(self, client: Any, filters: dict):
+        """
+        Retrieve original data from target sources.
+
+        Args:
+            client (Any): The client used to retrieve the results.
+            filters (dict): The filter criteria to apply.
+
+        Returns:
+            tuple: A tuple containing the results and the total count of results.
+        """
+        pass
+
+    @abstractmethod
+    def get_export_item(self, results):
+        """
+        Providing elements to build export data.
+
+        Args:
+            results (list): The results to process.
+
+        Yields:
+            generator: A generator that yields processed data items.
+        """
+        pass
+
+
+class AssignmentExportTargetHandler(ExportTargetHandler):
+    def validate_filter(self, value: dict, client: Any):
+        if 'project' not in value:
+            raise PydanticCustomError('missing_field', _('Project is required for Assignment.'))
+        try:
+            client.list_assignments(params=value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Assignment.'))
+        return value
+
+    def get_results(self, client: Any, filters: dict):
+        return client.list_assignments(params=filters, list_all=True)
+
+    def get_export_item(self, results):
+        for result in results:
+            yield {
+                'data': result['data'],
+                'files': result['file'],
+                'id': result['id'],
+            }
+
+
+class GroundTruthExportTargetHandler(ExportTargetHandler):
+    def validate_filter(self, value: dict, client: Any):
+        if 'ground_truth_dataset_version' not in value:
+            raise PydanticCustomError('missing_field', _('Ground Truth dataset version is required.'))
+        try:
+            client.get_ground_truth_version(value['ground_truth_dataset_version'])
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
+        return value
+
+    def get_results(self, client: Any, filters: dict):
+        filters['ground_truth_dataset_versions'] = filters.pop('ground_truth_dataset_version')
+        return client.list_ground_truth_events(params=filters, list_all=True)
+
+    def get_export_item(self, results):
+        for result in results:
+            files_key = next(iter(result['data_unit']['files']))
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'][files_key],
+                'id': result['ground_truth'],
+            }
+
+
+class TaskExportTargetHandler(ExportTargetHandler):
+    def validate_filter(self, value: dict, client: Any):
+        if 'project' not in value:
+            raise PydanticCustomError('missing_field', _('Project is required for Task.'))
+        try:
+            client.list_tasks(params=value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Task.'))
+        return value
+
+    def get_results(self, client: Any, filters: dict):
+        filters['expand'] = 'data_unit'
+        return client.list_tasks(params=filters, list_all=True)
+
+    def get_export_item(self, results):
+        for result in results:
+            files_key = next(iter(result['data_unit']['files']))
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'][files_key],
+                'id': result['id'],
+            }
+
+
+class TargetHandlerFactory:
+    @staticmethod
+    def get_handler(target: str) -> ExportTargetHandler:
+        if target == 'assignment':
+            return AssignmentExportTargetHandler()
+        elif target == 'ground_truth':
+            return GroundTruthExportTargetHandler()
+        elif target == 'task':
+            return TaskExportTargetHandler()
+        else:
+            raise ValueError(f'Unknown target: {target}')
+
+
+class ExportParams(BaseModel):
+    """
+    Parameters for the export action.
+
+    Attributes:
+        storage (int): The storage ID to save the exported data.
+        save_original_file (bool): Whether to save the original file.
+        path (str): The path to save the exported data.
+        target (str): The target source to export data from. (ex. ground_truth, assignment, task)
+        filter (dict): The filter criteria to apply.
+    """
+
+    storage: int
+    save_original_file: bool = True
+    path: str
+    target: Literal['assignment', 'ground_truth', 'task']
+    filter: dict
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('filter')
+    @staticmethod
+    def check_filter_by_target(value, info):
+        action = info.context['action']
+        client = action.client
+        target = action.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+        return handler.validate_filter(value, client)
+
+
+@register_action
+class ExportAction(Action):
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB
+    params_model = ExportParams
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+
+    def get_filtered_results(self, filters, handler):
+        """Get filtered target results."""
+        try:
+            result_list = handler.get_results(self.client, filters)
+            results = result_list[0]
+            count = result_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+        return results, count
+
+    def start(self):
+        filters = {'expand': 'data', **self.params['filter']}
+        target = self.params['target']
+        handler = TargetHandlerFactory.get_handler(target)
+
+        self.params['results'], self.params['count'] = self.get_filtered_results(filters, handler)
+        export_items = handler.get_export_item(self.params['results'])
+
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        return self.entrypoint(self.run, export_items, pathlib_cwd, **self.params)

{synapse_sdk-1.0.0a32 → synapse_sdk-1.0.0a34}/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -4,12 +4,12 @@ from pathlib import Path
 import requests
 
 
-def export(run, input_dataset, path_root, **params):
+def export(run, export_items, path_root, **params):
     """Executes the export task.
 
     Args:
         run : Execution object
-        input_dataset (generator):
+        export_items (generator):
             - data (dict): dm_schema_data information.
             - files (dict): File information. Includes file URL, original file path, metadata, etc.
             - id (int): ground_truth ID
@@ -42,11 +42,11 @@ def export(run, input_dataset, path_root, **params):
     total = params['count']
     # progress init
     run.set_progress(0, total, category='dataset_conversion')
-    for no, input_data in enumerate(input_dataset, start=1):
+    for no, export_item in enumerate(export_items, start=1):
         run.set_progress(no, total, category='dataset_conversion')
         if no == 1:
             run.log_message('Converting dataset.')
-        preprocessed_data = before_convert(input_data)
+        preprocessed_data = before_convert(export_item)
         converted_data = convert_data(preprocessed_data)
         final_data = after_convert(converted_data)
 
@@ -97,7 +97,7 @@ def get_original_file_pathlib(files):
     Returns:
         pathlib.Path: The original file path extracted from the metadata.
     """
-    return Path(next(iter(files.values()))['meta']['path_original'])
+    return Path(files['meta']['path_original'])
 
 
 def save_original_file(result, base_path, error_file_list):
@@ -108,7 +108,7 @@ def save_original_file(result, base_path, error_file_list):
         base_path (Path): The directory where the file will be saved.
         error_file_list (list): A list to store error files.
     """
-    file_url = next(iter(result['files'].values()))['url']
+    file_url = result['files']['url']
     file_name = get_original_file_pathlib(result['files']).name
     response = requests.get(file_url)
     try:

synapse_sdk-1.0.0a34/synapse_sdk/plugins/categories/neural_net/actions/deployment.py

@@ -0,0 +1,107 @@
+from packaging import version
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+
+
+@register_action
+class DeploymentAction(Action):
+    name = 'deployment'
+    category = PluginCategory.NEURAL_NET
+    method = RunMethod.JOB
+
+    def get_actor_options(self):
+        options = {'runtime_env': self.get_runtime_env()}
+        for option in ['num_cpus', 'num_gpus']:
+            option_value = self.params.get(option)
+            if option_value:
+                options[option] = option_value
+        return options
+
+    def start(self):
+        from ray import serve
+
+        from synapse_sdk.plugins.categories.neural_net.base.inference import app
+
+        self.ray_init()
+
+        ray_actor_options = self.get_actor_options()
+
+        if self.is_gradio_deployment:
+            from ray.serve.gradio_integrations import GradioServer
+
+            self.assert_gradio_version()
+
+            # GradioIngress differs from serve.ingress(app), thus the difference in self.entrypoint callable
+            try:
+                entrypoint = self.entrypoint().app
+            except (TypeError, ImportError):
+                raise ClientError(
+                    400,
+                    _(
+                        'Gradio app is not callable.'
+                        'Please ensure that your Deployment class defines a callable `app` function'
+                    ),
+                )
+
+            deployment = GradioServer.options(ray_actor_options=ray_actor_options).bind(entrypoint)
+        else:
+            deployment = serve.deployment(ray_actor_options=ray_actor_options)(
+                serve.ingress(app)(self.entrypoint)
+            ).bind(self.envs['SYNAPSE_PLUGIN_RUN_HOST'])
+
+        serve.delete(self.plugin_release.code)
+
+        # TODO add run object
+        serve.run(
+            deployment,
+            name=self.plugin_release.code,
+            route_prefix=f'/{self.plugin_release.checksum}',
+        )
+
+        # 백엔드에 ServeApplication 추가
+        serve_application = self.create_serve_application()
+        return {'serve_application': serve_application['id'] if serve_application else None}
+
+    def create_serve_application(self):
+        if self.job_id:
+            try:
+                serve_application = self.ray_client.get_serve_application(self.plugin_release.code)
+                return self.client.create_serve_application({
+                    'job': self.job_id,
+                    'status': serve_application['status'],
+                    'data': serve_application,
+                })
+            except ClientError:
+                pass
+        return None
+
+    @property
+    def is_gradio_deployment(self):
+        return self.config.get('gradio_app', False)
+
+    def assert_gradio_version(self):
+        """Assert gradio version is not greater than 3.50.2.
+        Ray Serve cannot pickle gradio endpoints, thus gradio version greater than 3.50.2 is not supported (SSE Issues)
+        """
+        GRADIO_VERSION_MAX_ALLOWED = '3.50.2'
+
+        gradio_installed = False
+        gradio_version = None
+        for req in self.requirements:
+            if req.startswith('gradio=='):
+                gradio_installed = True
+                gradio_version = req.split('==')[1]
+                break
+
+        assert gradio_installed, _(
+            'Gradio is not installed or version is not specified. Please install gradio==3.50.2 to use this feature.'
+        )
+
+        if version.parse(gradio_version) > version.parse(GRADIO_VERSION_MAX_ALLOWED):
+            raise AssertionError(
+                f'Gradio version {gradio_version} is greater than maximum allowed version {GRADIO_VERSION_MAX_ALLOWED}'
+            )

{synapse_sdk-1.0.0a32 → synapse_sdk-1.0.0a34}/synapse_sdk/plugins/categories/smart_tool/templates/config.yaml

@@ -6,3 +6,5 @@ actions:
     entrypoint: plugin.auto_label.MyAutoLabel
     model:
       neural_net__plugin__code: sam2
+    ui_schema: |
+      Dumped FormKit Schema for auto_label action

{synapse_sdk-1.0.0a32 → synapse_sdk-1.0.0a34}/synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Annotated, Dict, List
 
 from pydantic import AfterValidator, BaseModel, field_validator
@@ -6,6 +5,7 @@ from pydantic_core import PydanticCustomError
 
 from synapse_sdk.clients.exceptions import ClientError
 from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.clients.validators.collections import FileSpecificationValidator
 from synapse_sdk.i18n import gettext as _
 from synapse_sdk.plugins.categories.base import Action
 from synapse_sdk.plugins.categories.decorators import register_action
@@ -145,7 +145,7 @@ class UploadAction(Action):
 
         # Analyze Collection file specifications to determine the data structure for upload.
         self.run.set_progress(0, 1, category='analyze_collection')
-        file_specification_skeleton = self._analyze_collection()
+        file_specification_template = self._analyze_collection()
         self.run.set_progress(1, 1, category='analyze_collection')
 
         # Setup result dict.
@@ -153,7 +153,7 @@ class UploadAction(Action):
 
         # Organize data according to Collection file specification structure.
         organized_files = uploader.handle_upload_files()
-        if not self._validate_organized_files(file_specification_skeleton, organized_files):
+        if not self._validate_organized_files(file_specification_template, organized_files):
             self.run.log_message('Validate organized files failed.')
             return result
 
@@ -219,9 +219,10 @@ class UploadAction(Action):
         collection = client.get_dataset(collection_id)
         return collection['file_specifications']
 
-    def _validate_organized_files(self, file_specification_skeleton: Dict, organized_files: List) -> bool:
+    def _validate_organized_files(self, file_specification_template: Dict, organized_files: List) -> bool:
         """Validate organized files from Uploader."""
-        return True
+        validator = FileSpecificationValidator(file_specification_template, organized_files)
+        return validator.validate()
 
     def _upload_files(self, organized_files) -> List:
         """Upload files to synapse-backend.