synapse-sdk 1.0.0a30__py3-none-any.whl → 1.0.0a32__py3-none-any.whl

synapse_sdk/clients/backend/dataset.py

@@ -1,4 +1,6 @@
 from multiprocessing import Pool
+from pathlib import Path
+from typing import Dict, Optional
 
 from tqdm import tqdm
 
@@ -11,21 +13,59 @@ class DatasetClientMixin(BaseClient):
         path = 'datasets/'
         return self._list(path)
 
-    def create_data_file(self, file_path):
+    def get_dataset(self, dataset_id):
+        """Get dataset from synapse-backend.
+
+        Args:
+            dataset_id: The dataset id to get.
+        """
+        path = f'datasets/{dataset_id}/?expand=file_specifications'
+        return self._get(path)
+
+    def create_data_file(self, file_path: Path):
+        """Create data file to synapse-backend.
+
+        Args:
+            file_path: The file pathlib object to upload.
+        """
         path = 'data_files/'
         return self._post(path, files={'file': file_path})
 
     def create_data_units(self, data):
+        """Create data units to synapse-backend.
+
+        Args:
+            data: The data bindings to upload from create_data_file interface.
+        """
         path = 'data_units/'
         return self._post(path, data=data)
 
-    def import_dataset(self, dataset_id, dataset, project_id=None, batch_size=1000, process_pool=10):
+    def upload_dataset(
+        self,
+        dataset_id: int,
+        dataset: Dict,
+        project_id: Optional[int] = None,
+        batch_size: int = 1000,
+        process_pool: int = 10,
+    ):
+        """Upload dataset to synapse-backend.
+
+        Args:
+            dataset_id: The dataset id to upload the data to.
+            dataset: The dataset to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            project_id: The project id to upload the data to.
+            batch_size: The batch size to upload the data.
+            process_pool: The process pool to upload the data.
+        """
         # TODO validate dataset with schema
 
         params = [(data, dataset_id) for data in dataset]
 
         with Pool(processes=process_pool) as pool:
-            dataset = pool.starmap(self.import_data_file, tqdm(params))
+            dataset = pool.starmap(self.upload_data_file, tqdm(params))
 
         batches = get_batched_list(dataset, batch_size)
 
@@ -36,13 +76,25 @@ class DatasetClientMixin(BaseClient):
                 tasks_data = []
                 for data, data_unit in zip(batch, data_units):
                     task_data = {'project': project_id, 'data_unit': data_unit['id']}
-                    # TODO: 추후 import 시 Task data 저장 필요 시 해당 로직 추가 필요.
+                    # TODO: Additional logic needed here if task data storage is required during import.
 
                     tasks_data.append(task_data)
 
                 self.create_tasks(tasks_data)
 
-    def import_data_file(self, data, dataset_id):
+    def upload_data_file(self, data: Dict, dataset_id: int) -> Dict:
+        """Upload files to synapse-backend.
+
+        Args:
+            data: The data to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            dataset_id: The dataset id to upload the data to.
+
+        Returns:
+            Dict: The result of the upload.
+        """
         for name, path in data['files'].items():
             data_file = self.create_data_file(path)
             data['dataset'] = dataset_id

synapse_sdk/clients/base.py

@@ -40,37 +40,31 @@ class BaseClient:
         headers = self._get_headers()
         headers.update(kwargs.pop('headers', {}))
 
+        # List to store opened files to close after request
+        opened_files = []
+
         if method in ['post', 'put', 'patch']:
             # If files are included in the request, open them as binary files
             if kwargs.get('files') is not None:
-                opened_files = []  # List to store opened files
-                try:
-                    for name, file in kwargs['files'].items():
-                        # If file is a path string, bind it as a Path object and open
-                        if isinstance(file, str):
-                            opened_file = open(Path(file), mode='rb')
-                            kwargs['files'][name] = opened_file
-                            opened_files.append(opened_file)
-                        # If file is a Path object, open it directly
-                        elif isinstance(file, Path):
-                            opened_file = open(file, mode='rb')
-                            kwargs['files'][name] = opened_file
-                            opened_files.append(opened_file)
-                    if 'data' in kwargs:
-                        for name, value in kwargs['data'].items():
-                            if isinstance(value, dict):
-                                kwargs['data'][name] = json.dumps(value)
-                finally:
-                    # Close all opened files
-                    for opened_file in opened_files:
-                        opened_file.close()
-
+                for name, file in kwargs['files'].items():
+                    # Handle both string and Path object cases
+                    if isinstance(file, str):
+                        file = Path(file)
+                    if isinstance(file, Path):
+                        opened_file = file.open(mode='rb')
+                        kwargs['files'][name] = (file.name, opened_file)
+                        opened_files.append(opened_file)
+                if 'data' in kwargs:
+                    for name, value in kwargs['data'].items():
+                        if isinstance(value, dict):
+                            kwargs['data'][name] = json.dumps(value)
             else:
                 headers['Content-Type'] = 'application/json'
                 if 'data' in kwargs:
                     kwargs['data'] = json.dumps(kwargs['data'])
 
         try:
+            # Send request
             response = getattr(self.requests_session, method)(url, headers=headers, **kwargs)
             if not response.ok:
                 raise ClientError(
@@ -79,6 +73,10 @@ class BaseClient:
         except requests.ConnectionError:
             raise ClientError(408, f'{self.name} is not responding')
 
+        # Close all opened files
+        for opened_file in opened_files:
+            opened_file.close()
+
         return self._post_response(response)
 
     def _post_response(self, response):

synapse_sdk/plugins/categories/base.py

@@ -17,6 +17,30 @@ from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
 
 class Action:
+    """Base class for all plugin actions.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+        run_class (Run): The class to run the action.
+        params_model (BaseModel): The model to validate the params.
+        progress_categories (List[str]): The categories to update the progress.
+        params (Dict): The params to run the action.
+        plugin_config (Dict): The plugin config.
+        plugin_release (PluginRelease): The plugin release.
+        config (Dict): The action config.
+        requirements (List[str]): The requirements to install.
+        job_id (str): The job id.
+        direct (bool): The flag to run the action directly.
+        debug (bool): The flag to run the action in debug mode.
+        envs (Dict): The runtime envs.
+        run (Run): The run instance.
+
+    Raises:
+        ActionError: If the action fails.
+    """
+
     # class 변수
     name = None
     category = None
@@ -159,11 +183,19 @@ class Action:
         return getattr(self, f'start_by_{self.method.value}')()
 
     def start(self):
+        """Start the action.
+
+        TODO: Specify the return type of start method for overrided methods.
+        """
         if self.method == RunMethod.JOB:
             return self.entrypoint(self.run, **self.params)
         return self.entrypoint(**self.params)
 
     def start_by_task(self):
+        """Ray Task based execution.
+
+        * A task method that simply executes the entrypoint without job management functionality.
+        """
         import ray
         from ray.exceptions import RayTaskError
 
@@ -195,6 +227,10 @@ class Action:
             raise ActionError(e.cause)
 
     def start_by_job(self):
+        """Ray Job based execution.
+
+        * Executes the entrypoint with Ray job. Ray job manages the entrypoint execution and stores the results.
+        """
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -215,6 +251,10 @@ class Action:
         )
 
     def start_by_restapi(self):
+        """Ray Serve based execution.
+
+        * This method executes a Fastapi endpoint defined within the Plugin.
+        """
         path = self.params.pop('path', '')
         method = self.params.pop('method')
 

synapse_sdk/plugins/categories/export/actions/export.py

@@ -61,11 +61,8 @@ class ExportAction(Action):
             }
 
     def get_filtered_results(self):
-        """Get filtered ground truth."""
-        ground_truth_dataset_version = self.params['ground_truth_dataset_version']
-        self.params['filter']['ground_truth_dataset'] = self.client.get_ground_truth_version(
-            ground_truth_dataset_version
-        )['ground_truth_dataset']
+        """Get filtered ground truth events."""
+        self.params['filter']['ground_truth_dataset_versions'] = self.params['ground_truth_dataset_version']
         filters = {'expand': 'data', **self.params['filter']}
 
         try:

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -1,9 +1,8 @@
 import json
+from pathlib import Path
 
 import requests
 
-from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
-
 
 def export(run, input_dataset, path_root, **params):
     """Executes the export task.
@@ -14,7 +13,7 @@ def export(run, input_dataset, path_root, **params):
             - data (dict): dm_schema_data information.
             - files (dict): File information. Includes file URL, original file path, metadata, etc.
             - id (int): ground_truth ID
-        path_root : Save path
+        path_root : pathlib object, the path to export
         **params: Additional parameters
 
     Returns:
@@ -24,10 +23,11 @@ def export(run, input_dataset, path_root, **params):
     path_root.mkdir(parents=True, exist_ok=True)
     run.log_message('Starting export process.')
 
-    # results: Contains all information fetched through the list API.
-    results = params.get('results', [])
+    # results contains all information fetched through the list API.
+    # example:
+    #   params.get('results', [])
 
-    save_original_file = params.get('save_original_file')
+    save_original_file_flag = params.get('save_original_file')
     errors_json_file_list = []
     errors_original_file_list = []
 
@@ -39,21 +39,28 @@ def export(run, input_dataset, path_root, **params):
     origin_files_output_path = path_root / 'origin_files'
     origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
-    total = len(results)
-    for no, input_data in enumerate(input_dataset):
+    total = params['count']
+    # progress init
+    run.set_progress(0, total, category='dataset_conversion')
+    for no, input_data in enumerate(input_dataset, start=1):
         run.set_progress(no, total, category='dataset_conversion')
+        if no == 1:
+            run.log_message('Converting dataset.')
         preprocessed_data = before_convert(input_data)
         converted_data = convert_data(preprocessed_data)
         final_data = after_convert(converted_data)
 
         # Call if original file extraction is needed
-        if save_original_file:
+        if save_original_file_flag:
+            if no == 1:
+                run.log_message('Saving original file.')
             save_original_file(final_data, origin_files_output_path, errors_original_file_list)
 
         # Extract data as JSON files
+        if no == 1:
+            run.log_message('Saving json file.')
         save_as_json(final_data, json_output_path, errors_json_file_list)
 
-    run.log_message('Saving converted dataset.')
     run.end_log()
 
     # Save error list files
@@ -62,7 +69,7 @@ def export(run, input_dataset, path_root, **params):
         with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
             json.dump(export_error_file, f, indent=4, ensure_ascii=False)
 
-    return {'export_path': path_root}
+    return {'export_path': str(path_root)}
 
 
 def convert_data(data):
@@ -80,19 +87,29 @@ def after_convert(data):
     return data
 
 
-def save_original_file(result, base_path, error_file_list):
-    """Saves the original file.
+def get_original_file_pathlib(files):
+    """Retrieve the original file path from the given file information.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
+        files (dict): A dictionary containing file information, including file URL,
+                      original file path, metadata, etc.
 
     Returns:
-        base_path (str): Save path
+        pathlib.Path: The original file path extracted from the metadata.
+    """
+    return Path(next(iter(files.values()))['meta']['path_original'])
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
     file_url = next(iter(result['files'].values()))['url']
-    file_name = get_original_file_path(result['files']).name
+    file_name = get_original_file_pathlib(result['files']).name
     response = requests.get(file_url)
     try:
         with (base_path / file_name).open('wb') as file:
@@ -100,27 +117,20 @@ def save_original_file(result, base_path, error_file_list):
     except Exception as e:
         error_file_list.append([file_name, str(e)])
 
-    return base_path
-
 
 def save_as_json(result, base_path, error_file_list):
     """Saves the data as a JSON file.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
-
-    Returns:
-        base_path (str): Save path
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
     # Default save file name: original file name
-    file_name = get_original_file_path(result['files']).stem
+    file_name = get_original_file_pathlib(result['files']).stem
     json_data = result['data']
     try:
         with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
             json.dump(json_data, f, indent=4, ensure_ascii=False)
     except Exception as e:
         error_file_list.append([f'{file_name}.json', str(e)])
-
-    return base_path

synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -51,7 +51,7 @@ class TrainParams(BaseModel):
                 params={
                     'ids_ex': action.job_id,
                     'category': 'neural_net',
-                    'action': 'train',
+                    'job__action': 'train',
                     'is_active': True,
                     'params': f'name:{value}',
                 },

synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -0,0 +1,292 @@
+from enum import Enum
+from typing import Annotated, Dict, List
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+from synapse_sdk.utils.pydantic.validators import non_blank
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class UploadRun(Run):
+    pass
+
+
+class UploadParams(BaseModel):
+    """Upload action parameters.
+
+    Args:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
+        checkpoint (int | None): The checkpoint of the action.
+        path (str): The path of the action.
+        storage (int): The storage of the action.
+        collection (int): The collection of the action.
+        project (int | None): The project of the action.
+        is_generate_tasks (bool): The flag to generate tasks.
+        is_generate_ground_truths (bool): The flag to generate ground truths
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None
+    path: str
+    storage: int
+    collection: int
+    project: int | None
+    is_generate_tasks: bool = False
+    is_generate_ground_truths: bool = False
+
+    @field_validator('storage', mode='before')
+    @classmethod
+    def check_storage_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend storage exists.
+
+        TODO: Need to define validation method naming convention.
+        TODO: Need to make validation method reusable.
+        """
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking storage exists.'))
+        return value
+
+    @field_validator('collection', mode='before')
+    @classmethod
+    def check_collection_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend collection exists."""
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_dataset(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking collection exists.'))
+        return value
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend project exists."""
+        if not value:
+            return value
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking project exists.'))
+        return value
+
+
+@register_action
+class UploadAction(Action):
+    """Upload action class.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+
+    Progress Categories:
+        analyze_collection: The progress category for the analyze collection process.
+        data_file_upload: The progress category for the upload process.
+        generate_data_units: The progress category for the generate data units process.
+        generate_tasks: The progress category for the generate tasks process.
+        generate_ground_truths: The progress category for the generate ground truths process.
+    """
+
+    name = 'upload'
+    category = PluginCategory.UPLOAD
+    method = RunMethod.JOB
+    progress_categories = {
+        'analyze_collection': {
+            'proportion': 5,
+        },
+        'upload_data_files': {
+            'proportion': 35,
+        },
+        'generate_data_units': {
+            'proportion': 20,
+        },
+        'generate_tasks': {
+            'proportion': 20,
+        },
+        'generate_ground_truths': {
+            'proportion': 20,
+        },
+    }
+
+    def get_uploader(self, path):
+        """Get uploader from entrypoint."""
+        return self.entrypoint(self.run, path)
+
+    def start(self) -> Dict:
+        """Start upload process.
+
+        Returns:
+            Dict: The result of the upload process.
+        """
+        # Setup path object with path and storage.
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+
+        # Initialize uploader.
+        uploader = self.get_uploader(pathlib_cwd)
+
+        # Analyze Collection file specifications to determine the data structure for upload.
+        self.run.set_progress(0, 1, category='analyze_collection')
+        file_specification_skeleton = self._analyze_collection()
+        self.run.set_progress(1, 1, category='analyze_collection')
+
+        # Setup result dict.
+        result = {}
+
+        # Organize data according to Collection file specification structure.
+        organized_files = uploader.handle_upload_files()
+        if not self._validate_organized_files(file_specification_skeleton, organized_files):
+            self.run.log_message('Validate organized files failed.')
+            return result
+
+        # Upload files to synapse-backend.
+        organized_files_count = len(organized_files)
+        if not organized_files_count:
+            self.run.log_message('Files not found on the path.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, organized_files_count, category='upload_data_files')
+        self.run.log_message('Uploading data files...')
+        result['uploaded_files'] = self._upload_files(organized_files)
+        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+        self.run.log_message('Upload data files completed.')
+
+        # Generate data units for the uploaded data.
+        upload_result_count = len(result['uploaded_files'])
+        if not upload_result_count:
+            self.run.log_message('No files were uploaded.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, upload_result_count, category='generate_data_units')
+        generated_data_units = self._generate_data_units(result['uploaded_files'])
+        result['generated_data_units'] = generated_data_units
+        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+
+        # Setup task with uploaded synapse-backend data units.
+        if not len(generated_data_units):
+            self.run.log_message('No data units were generated.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, 1, category='generate_tasks')
+        if self.config['options']['allow_generate_tasks'] and self.params['is_generate_tasks']:
+            self.run.log_message('Generating tasks with data files...')
+            self._generate_tasks(generated_data_units)
+            self.run.log_message('Generating tasks completed')
+        else:
+            self.run.log_message('Generating tasks process has passed.')
+
+        self.run.set_progress(1, 1, category='generate_tasks')
+
+        # Generate ground truths for the uploaded data.
+        # TODO: Need to add ground truths generation logic later.
+        self.run.set_progress(0, 1, category='generate_ground_truths')
+        if self.config['options']['allow_generate_ground_truths'] and self.params['is_generate_ground_truths']:
+            self.run.log_message('Generating ground truths...')
+            self._generate_ground_truths()
+            self.run.log_message('Generating ground truths completed')
+        else:
+            self.run.log_message('Generating ground truths process has passed.')
+        self.run.set_progress(1, 1, category='generate_ground_truths')
+
+        return result
+
+    def _analyze_collection(self) -> Dict:
+        """Analyze Synapse Collection Specifications.
+
+        Returns:
+            Dict: The file specifications of the collection.
+        """
+        client = self.run.client
+        collection_id = self.params['collection']
+        collection = client.get_dataset(collection_id)
+        return collection['file_specifications']
+
+    def _validate_organized_files(self, file_specification_skeleton: Dict, organized_files: List) -> bool:
+        """Validate organized files from Uploader."""
+        return True
+
+    def _upload_files(self, organized_files) -> List:
+        """Upload files to synapse-backend.
+
+        Returns:
+            Dict: The result of the upload.
+        """
+        client = self.run.client
+        collection_id = self.params['collection']
+        upload_result = []
+        organized_files_count = len(organized_files)
+        current_progress = 0
+        for organized_file in organized_files:
+            upload_result.append(client.upload_data_file(organized_file, collection_id))
+            self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
+            current_progress += 1
+        return upload_result
+
+    def _generate_data_units(self, uploaded_files: List) -> List:
+        """Generate data units for the uploaded data.
+
+        TODO: make batch size configurable.
+
+        Returns:
+            Dict: The result of the generate data units process.
+        """
+        client = self.run.client
+
+        generation_result = []
+        current_progress = 0
+        batches = get_batched_list(uploaded_files, 100)
+        batches_count = len(batches)
+        for batch in batches:
+            generation_result.append(client.create_data_units(batch))
+            self.run.set_progress(current_progress, batches_count, category='generate_data_units')
+            current_progress += 1
+        return generation_result
+
+    def _generate_tasks(self, generated_data_units: List):
+        """Setup task with uploaded synapse-backend data units.
+
+        TODO: make batch size configurable.
+        """
+
+        # Prepare batches for processing
+        client = self.run.client
+        project_id = self.params['project']
+        current_progress = 0
+
+        # Generate tasks
+        generated_data_units_count = len(generated_data_units)
+        for data_units in generated_data_units:
+            tasks_data = []
+            for data_unit in data_units:
+                task_data = {'project': project_id, 'data_unit': data_unit['id']}
+                tasks_data.append(task_data)
+
+            if tasks_data:
+                client.create_tasks(tasks_data)
+
+            self.run.set_progress(current_progress, generated_data_units_count, category='generate_tasks')
+            current_progress += 1
+
+    def _generate_ground_truths(self):
+        """Generate ground truths for the uploaded data.
+
+        TODO: Need to add ground truths generation logic later.
+        """

synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -0,0 +1,6 @@
+actions:
+  upload:
+    entrypoint: plugin.upload.Uploader
+    options:
+      allow_generate_tasks: false
+      allow_generate_ground_truths: false

synapse_sdk/plugins/categories/upload/templates/plugin/upload.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+from typing import List
+
+
+class Uploader:
+    """Plugin upload action class.
+
+    * Organize, upload, setup task, generate ground truths for the uploaded data.
+    """
+
+    def __init__(self, run, path: Path, *args, **kwargs):
+        """Initialize the plugin upload action class.
+
+        Args:
+            run: Plugin run object.
+            path: pathlib object by upload target destination path.
+        """
+        self.run = run
+        self.path = path
+
+    def handle_upload_files(self) -> List:
+        """Handle upload files.
+
+        * Organize data according to collection file specification structure.
+        * Structure files according to the file specification of the target collection.
+
+        Returns:
+            List: List of dictionaries containing 'files' and 'meta'.
+
+        Examples:
+            [
+              {
+          "files": {
+              'image_1': image_1_pathlib_object,
+              'image_2': image_2_pathlib_object,
+              'meta_1': meta_1_pathlib_object,
+          },
+          "meta": {
+            "key": "value"
+          }
+              }
+            ]
+        """
+        return []

synapse_sdk/plugins/enums.py

@@ -2,6 +2,8 @@ from enum import Enum
 
 
 class RunMethod(Enum):
+    """Plugin Execution Methods."""
+
     JOB = 'job'
     TASK = 'task'
     RESTAPI = 'restapi'
@@ -10,7 +12,7 @@ class RunMethod(Enum):
 class PluginCategory(Enum):
     NEURAL_NET = 'neural_net'
     EXPORT = 'export'
-    IMPORT = 'import'
+    UPLOAD = 'upload'
     SMART_TOOL = 'smart_tool'
     POST_ANNOTATION = 'post_annotation'
     PRE_ANNOTATION = 'pre_annotation'

synapse_sdk/plugins/models.py

@@ -87,6 +87,15 @@ class PluginRelease:
 
 
 class Run:
+    """Run class for manage plugin run istance.
+
+    Attrs:
+        job_id: plugin run job id
+        context: plugin run context
+        client: backend client for communicate with backend
+        logger: logger for log plugin run events
+    """
+
     logger = None
     job_id = None
     context = None
@@ -111,6 +120,13 @@ class Run:
             self.logger = ConsoleLogger(**kwargs)
 
     def set_progress(self, current, total, category=''):
+        """Set progress for plugin run.
+
+        Args:
+            current: current progress value
+            total: total progress value
+            category: progress category
+        """
         self.logger.set_progress(current, total, category)
 
     def log(self, event, data, file=None):

synapse_sdk/utils/storage/__init__.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from urllib.parse import urlparse
 
 from synapse_sdk.i18n import gettext as _
@@ -15,6 +16,15 @@ def get_storage(connection_param: str | dict):
     return STORAGE_PROVIDERS[storage_scheme](connection_param)
 
 
-def get_pathlib(storage_config, path_root):
+def get_pathlib(storage_config: str | dict, path_root: str) -> Path:
+    """Get pathlib object with synapse-backend storage config.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path_root (str): The path root.
+
+    Returns:
+        pathlib.Path: The pathlib object.
+    """
     storage_class = get_storage(storage_config)
     return storage_class.get_pathlib(path_root)

{synapse_sdk-1.0.0a30.dist-info → synapse_sdk-1.0.0a32.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a30
+Version: 1.0.0a32
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -21,6 +21,7 @@ Requires-Dist: universal-pathlib
 Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
+Dynamic: license-file
 
 This is the SDK to develop synapse plugins
 

{synapse_sdk-1.0.0a30.dist-info → synapse_sdk-1.0.0a32.dist-info}/RECORD

@@ -21,7 +21,7 @@ synapse_sdk/cli/plugin/create.py,sha256=HpYTpohV1NbSrULaVUlc4jWLWznPrx7glgydTM3s
 synapse_sdk/cli/plugin/publish.py,sha256=sIl1wiuSC3lAUpE3rOF4UDKDy2G5EVLlelMjk2aT05g,1221
 synapse_sdk/cli/plugin/run.py,sha256=xz5LRm3zh8Y9DMjw5FFRFVRWSCWtYfZJskfCmrPikaQ,2598
 synapse_sdk/clients/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/clients/base.py,sha256=l4_9BvqKykE9EiW9KygTRBf4tFUZLutG4CfwFIo2LRg,4710
+synapse_sdk/clients/base.py,sha256=m3zEX1wv_sSxa-jBCnE-9Q5-f71o1TdODkWHRE9Vo-E,4403
 synapse_sdk/clients/exceptions.py,sha256=ylv7x10eOp4aA3a48jwonnvqvkiYwzJYXjkVkRTAjwk,220
 synapse_sdk/clients/utils.py,sha256=8pPJTdzHiRPSbZMoQYHAgR2BAMO6u_R_jMV6a2p34iQ,392
 synapse_sdk/clients/agent/__init__.py,sha256=Pz8_iTbIbnb7ywGJ3feqoZVmO2I3mEbwpWsISIxh0BU,1968
@@ -31,20 +31,20 @@ synapse_sdk/clients/agent/service.py,sha256=s7KuPK_DB1nr2VHrigttV1WyFonaGHNrPvU8
 synapse_sdk/clients/backend/__init__.py,sha256=aozhPhvRTPHz1P90wxEay07B-Ct4vj_yTw5H9_PJEBE,1105
 synapse_sdk/clients/backend/annotation.py,sha256=eZc5EidgR_RfMGwvv1r1_mLkPdRd8e52c4zuuMjMX34,979
 synapse_sdk/clients/backend/core.py,sha256=5XAOdo6JZ0drfk-FMPJ96SeTd9oja-VnTwzGXdvK7Bg,1027
-synapse_sdk/clients/backend/dataset.py,sha256=w7izflbTjHKysiDl7ia7MAO391_dzN2ofK40A7QwtBQ,1721
+synapse_sdk/clients/backend/dataset.py,sha256=11R5LuTva9jgXatxQAlKy7UEJmwIWzTsLVdFf3MZ9F8,3400
 synapse_sdk/clients/backend/integration.py,sha256=MCfeChpLySqlVRc1aZxCfDpQiRH--pfevkCdJDCNZEQ,2506
 synapse_sdk/clients/backend/ml.py,sha256=JoPH9Ly2E3HJ7S5mdGLtcGq7ruQVVrYfWArogwZLlms,1193
 synapse_sdk/clients/ray/__init__.py,sha256=9ZSPXVVxlJ8Wp8ku7l021ENtPjVrGgQDgqifkkVAXgM,187
 synapse_sdk/clients/ray/core.py,sha256=a4wyCocAma2HAm-BHlbZnoVbpfdR-Aad2FM0z6vPFvw,731
 synapse_sdk/clients/ray/serve.py,sha256=rbCpXZYWf0oP8XJ9faa9QFNPYU7h8dltIG8xn9ZconY,907
 synapse_sdk/plugins/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/enums.py,sha256=s59P6Oz2WAK9IX-kLVhNOvNKYJifKlWBhPpZbc9-ttE,486
+synapse_sdk/plugins/enums.py,sha256=ibixwqA3sCNSriG1jAtL54JQc_Zwo3MufwYUqGhVncc,523
 synapse_sdk/plugins/exceptions.py,sha256=Qs7qODp_RRLO9y2otU2T4ryj5LFwIZODvSIXkAh91u0,691
-synapse_sdk/plugins/models.py,sha256=7En1biVK_7kR8aI-3I-kJ-lXbveRRobppMGOeFd3ZpU,3709
+synapse_sdk/plugins/models.py,sha256=njTQIT-c2d7TsqAN__q1aoYm8hLEPC7ludTj665iN-4,4148
 synapse_sdk/plugins/upload.py,sha256=VJOotYMayylOH0lNoAGeGHRkLdhP7jnC_A0rFQMvQpQ,3228
 synapse_sdk/plugins/utils.py,sha256=4_K6jIl0WrsXOEhFp94faMOriSsddOhIiaXcawYYUUA,3300
 synapse_sdk/plugins/categories/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/base.py,sha256=_bNB_TeCN1hIKeDLxb3Cj9UAGO8BLju97__pLN625A8,8500
+synapse_sdk/plugins/categories/base.py,sha256=cfk7mgidCBnldZ8nCo_Nq_qzP9C1LJYgSXWtYT6hJDw,10046
 synapse_sdk/plugins/categories/decorators.py,sha256=Gw6T-UHwpCKrSt596X-g2sZbY_Z1zbbogowClj7Pr5Q,518
 synapse_sdk/plugins/categories/registry.py,sha256=KdQR8SUlLT-3kgYzDNWawS1uJnAhrcw2j4zFaTpilRs,636
 synapse_sdk/plugins/categories/templates.py,sha256=FF5FerhkZMeW1YcKLY5cylC0SkWSYdJODA_Qcm4OGYQ,887
@@ -56,20 +56,16 @@ synapse_sdk/plugins/categories/data_validation/templates/plugin/__init__.py,sha2
 synapse_sdk/plugins/categories/data_validation/templates/plugin/validation.py,sha256=90I5boUpEXvO3mEuKKBs528ls2A4h8Iw4ReOID2h00Y,139
 synapse_sdk/plugins/categories/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/export/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/actions/export.py,sha256=GNmsrJncf7WaxNfnN9I1sNiYYxMbqPhcbpnDcBQC1DM,3062
-synapse_sdk/plugins/categories/export/actions/utils.py,sha256=ixilT_VtgwtbX9kbBTrIybbAyW1kPLhNEkOHABKmZ88,131
+synapse_sdk/plugins/categories/export/actions/export.py,sha256=45hnvM3myFgRJyGpC_jpDu_VaDC-iaNCqy4DcH24eDU,2926
 synapse_sdk/plugins/categories/export/templates/config.yaml,sha256=N7YmnFROb3s3M35SA9nmabyzoSb5O2t2TRPicwFNN2o,56
 synapse_sdk/plugins/categories/export/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=422HPdxCTor0PlOwSw7J7-HyD3i0kq_1D_VjKegkr8c,3904
-synapse_sdk/plugins/categories/import/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/import/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-synapse_sdk/plugins/categories/import/actions/import.py,sha256=URn6TOp081odMT5D4NlZ2XEcyKelJx8fxzdoKSkXSAI,320
+synapse_sdk/plugins/categories/export/templates/plugin/export.py,sha256=cSUhgY5FDqMWVj4XlrOfzDN8q-NxOAvB3BTMXdqd-Eg,4604
 synapse_sdk/plugins/categories/neural_net/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/actions/deployment.py,sha256=Wmi7in_Mgizt1d5XcDR080h1CIMWKh2_mjub9N380qA,1917
 synapse_sdk/plugins/categories/neural_net/actions/inference.py,sha256=0a655ELqNVjPFZTJDiw4EUdcMCPGveUEKyoYqpwMFBU,1019
 synapse_sdk/plugins/categories/neural_net/actions/test.py,sha256=JY25eg-Fo6WbgtMkGoo_qNqoaZkp3AQNEypJmeGzEog,320
-synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=CBg8ky2-bvS5pZlzuf49HcdpyronHkSugwq0UeHhiKg,5401
+synapse_sdk/plugins/categories/neural_net/actions/train.py,sha256=kve6iTCg2kUeavMQTR2JFuoYDu-QWZFFlB58ZICQtdM,5406
 synapse_sdk/plugins/categories/neural_net/base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/neural_net/base/inference.py,sha256=R5DASI6-5vzsjDOYxqeGGMBjnav5qHF4hNJT8zNUR3I,1097
 synapse_sdk/plugins/categories/neural_net/templates/config.yaml,sha256=vSTHgKzi7sHzsuUvhOT001hzR6JBpPCue5nECmruEnI,433
@@ -95,6 +91,12 @@ synapse_sdk/plugins/categories/smart_tool/actions/auto_label.py,sha256=fHiqA8ntm
 synapse_sdk/plugins/categories/smart_tool/templates/config.yaml,sha256=7bvb4M1PLaoTOVDYF05L7yb8ix4rZOrAnEuZ-thMsyo,206
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 synapse_sdk/plugins/categories/smart_tool/templates/plugin/auto_label.py,sha256=eevNg0nOcYFR4z_L_R-sCvVOYoLWSAH1jwDkAf3YCjY,320
+synapse_sdk/plugins/categories/upload/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/upload/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/upload/actions/upload.py,sha256=V7gArcj7FZO8AT_ihUGodrc_7q49Q7j9BKc9DvoYOYQ,11178
+synapse_sdk/plugins/categories/upload/templates/config.yaml,sha256=0PhB2uD-9ufavZs7EiF6xj4aBgZuif9mFFGGfzG7HuY,147
+synapse_sdk/plugins/categories/upload/templates/plugin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+synapse_sdk/plugins/categories/upload/templates/plugin/upload.py,sha256=dnK8gy33GjG5ettayawDJv1gM3xCm1K6lM-PfeeTjQw,1163
 synapse_sdk/plugins/templates/cookiecutter.json,sha256=NxOWk9A_v1pO0Ny4IYT9Cj5iiJ16--cIQrGC67QdR0I,396
 synapse_sdk/plugins/templates/hooks/post_gen_project.py,sha256=jqlYkY1O2TxIR-Vh3gnwILYy8k-D39Xx66d2KNQVMCs,147
 synapse_sdk/plugins/templates/hooks/pre_prompt.py,sha256=aOAMM623s0sKFGjTZaotAOYFvsNMxeii4tPyhOAFKVE,539
@@ -117,15 +119,15 @@ synapse_sdk/utils/pydantic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 synapse_sdk/utils/pydantic/config.py,sha256=1vYOcUI35GslfD1rrqhFkNXXJOXt4IDqOPSx9VWGfNE,123
 synapse_sdk/utils/pydantic/errors.py,sha256=0v0T12eQBr1KrFiEOBu6KMaPK4aPEGEC6etPJGoR5b4,1061
 synapse_sdk/utils/pydantic/validators.py,sha256=G47P8ObPhsePmd_QZDK8EdPnik2CbaYzr_N4Z6En8dc,193
-synapse_sdk/utils/storage/__init__.py,sha256=oDRyXwwp9xxdCkNtHzDF2-cUwv6EMhKt4rWYbyAKaEk,689
+synapse_sdk/utils/storage/__init__.py,sha256=AOPEo1_4Tssr3X0_pu5cbmJP_V5ywx9qSocIzZTc2kM,1005
 synapse_sdk/utils/storage/registry.py,sha256=WaSN9SJR7s9sZgmTVl5k4mLFz-9R6X4ii82wefxs95A,335
 synapse_sdk/utils/storage/providers/__init__.py,sha256=NM9yRIWcPkH53DeNHgIhH9zaDFK8SJv0KptP1Afulyw,1125
 synapse_sdk/utils/storage/providers/gcp.py,sha256=i2BQCu1Kej1If9SuNr2_lEyTcr5M_ncGITZrL0u5wEA,363
 synapse_sdk/utils/storage/providers/s3.py,sha256=r94aUGVNf8yxihU0lN62yaXdxSS7P783_sfD-hCiK40,1191
 synapse_sdk/utils/storage/providers/sftp.py,sha256=TUQXkKJf0-fh8NhGC_1zzqI4autFNHZVCqngwkQ1aD4,523
-synapse_sdk-1.0.0a30.dist-info/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
-synapse_sdk-1.0.0a30.dist-info/METADATA,sha256=CZ9X66Gy-UkyIZic1g3FSh9kfCVJ-lWSkvON9qox20M,1138
-synapse_sdk-1.0.0a30.dist-info/WHEEL,sha256=jB7zZ3N9hIM9adW7qlTAyycLYW9npaWKLRzaoVcLKcM,91
-synapse_sdk-1.0.0a30.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
-synapse_sdk-1.0.0a30.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
-synapse_sdk-1.0.0a30.dist-info/RECORD,,
+synapse_sdk-1.0.0a32.dist-info/licenses/LICENSE,sha256=bKzmC5YAg4V1Fhl8OO_tqY8j62hgdncAkN7VrdjmrGk,1101
+synapse_sdk-1.0.0a32.dist-info/METADATA,sha256=jJsMUwX1BNln7bpCx7CGxkQQuE6UCCvSk4P2i9as7x4,1160
+synapse_sdk-1.0.0a32.dist-info/WHEEL,sha256=tTnHoFhvKQHCh4jz3yCn0WPTYIy7wXx3CJtJ7SJGV7c,91
+synapse_sdk-1.0.0a32.dist-info/entry_points.txt,sha256=VNptJoGoNJI8yLXfBmhgUefMsmGI0m3-0YoMvrOgbxo,48
+synapse_sdk-1.0.0a32.dist-info/top_level.txt,sha256=ytgJMRK1slVOKUpgcw3LEyHHP7S34J6n_gJzdkcSsw8,12
+synapse_sdk-1.0.0a32.dist-info/RECORD,,

{synapse_sdk-1.0.0a30.dist-info → synapse_sdk-1.0.0a32.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.2)
+Generator: setuptools (77.0.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

synapse_sdk/plugins/categories/export/actions/utils.py

@@ -1,5 +0,0 @@
-from pathlib import Path
-
-
-def get_original_file_path(files):
-    return Path(next(iter(files.values()))['meta']['path_original'])

synapse_sdk/plugins/categories/import/actions/import.py

@@ -1,10 +0,0 @@
-from synapse_sdk.plugins.categories.base import Action
-from synapse_sdk.plugins.categories.decorators import register_action
-from synapse_sdk.plugins.enums import PluginCategory, RunMethod
-
-
-@register_action
-class ImportAction(Action):
-    name = 'import'
-    category = PluginCategory.IMPORT
-    method = RunMethod.JOB