synapse-sdk 1.0.0a31__py3-none-any.whl → 1.0.0a33__py3-none-any.whl

synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -1,20 +1,19 @@
 import json
+from pathlib import Path
 
 import requests
 
-from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
 
-
-def export(run, input_dataset, path_root, **params):
+def export(run, export_items, path_root, **params):
     """Executes the export task.
 
     Args:
         run : Execution object
-        input_dataset (generator):
+        export_items (generator):
             - data (dict): dm_schema_data information.
             - files (dict): File information. Includes file URL, original file path, metadata, etc.
             - id (int): ground_truth ID
-        path_root : Save path
+        path_root : pathlib object, the path to export
         **params: Additional parameters
 
     Returns:
@@ -24,10 +23,11 @@ def export(run, input_dataset, path_root, **params):
     path_root.mkdir(parents=True, exist_ok=True)
     run.log_message('Starting export process.')
 
-    # results: Contains all information fetched through the list API.
-    results = params.get('results', [])
+    # results contains all information fetched through the list API.
+    # example:
+    #   params.get('results', [])
 
-    save_original_file = params.get('save_original_file')
+    save_original_file_flag = params.get('save_original_file')
     errors_json_file_list = []
     errors_original_file_list = []
 
@@ -39,21 +39,28 @@ def export(run, input_dataset, path_root, **params):
     origin_files_output_path = path_root / 'origin_files'
     origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
-    total = len(results)
-    for no, input_data in enumerate(input_dataset):
+    total = params['count']
+    # progress init
+    run.set_progress(0, total, category='dataset_conversion')
+    for no, export_item in enumerate(export_items, start=1):
         run.set_progress(no, total, category='dataset_conversion')
-        preprocessed_data = before_convert(input_data)
+        if no == 1:
+            run.log_message('Converting dataset.')
+        preprocessed_data = before_convert(export_item)
         converted_data = convert_data(preprocessed_data)
         final_data = after_convert(converted_data)
 
         # Call if original file extraction is needed
-        if save_original_file:
+        if save_original_file_flag:
+            if no == 1:
+                run.log_message('Saving original file.')
             save_original_file(final_data, origin_files_output_path, errors_original_file_list)
 
         # Extract data as JSON files
+        if no == 1:
+            run.log_message('Saving json file.')
         save_as_json(final_data, json_output_path, errors_json_file_list)
 
-    run.log_message('Saving converted dataset.')
     run.end_log()
 
     # Save error list files
@@ -62,7 +69,7 @@ def export(run, input_dataset, path_root, **params):
         with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
             json.dump(export_error_file, f, indent=4, ensure_ascii=False)
 
-    return {'export_path': path_root}
+    return {'export_path': str(path_root)}
 
 
 def convert_data(data):
@@ -80,19 +87,29 @@ def after_convert(data):
     return data
 
 
-def save_original_file(result, base_path, error_file_list):
-    """Saves the original file.
+def get_original_file_pathlib(files):
+    """Retrieve the original file path from the given file information.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
+        files (dict): A dictionary containing file information, including file URL,
+                      original file path, metadata, etc.
 
     Returns:
-        base_path (str): Save path
+        pathlib.Path: The original file path extracted from the metadata.
+    """
+    return Path(files['meta']['path_original'])
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
-    file_url = next(iter(result['files'].values()))['url']
-    file_name = get_original_file_path(result['files']).name
+    file_url = result['files']['url']
+    file_name = get_original_file_pathlib(result['files']).name
     response = requests.get(file_url)
     try:
         with (base_path / file_name).open('wb') as file:
@@ -100,27 +117,20 @@ def save_original_file(result, base_path, error_file_list):
     except Exception as e:
         error_file_list.append([file_name, str(e)])
 
-    return base_path
-
 
 def save_as_json(result, base_path, error_file_list):
     """Saves the data as a JSON file.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
-
-    Returns:
-        base_path (str): Save path
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
     # Default save file name: original file name
-    file_name = get_original_file_path(result['files']).stem
+    file_name = get_original_file_pathlib(result['files']).stem
     json_data = result['data']
     try:
         with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
             json.dump(json_data, f, indent=4, ensure_ascii=False)
     except Exception as e:
         error_file_list.append([f'{file_name}.json', str(e)])
-
-    return base_path

synapse_sdk/plugins/categories/smart_tool/templates/config.yaml

@@ -6,3 +6,5 @@ actions:
     entrypoint: plugin.auto_label.MyAutoLabel
     model:
       neural_net__plugin__code: sam2
+    ui_schema: |
+      Dumped FormKit Schema for auto_label action

synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -0,0 +1,292 @@
+from enum import Enum
+from typing import Annotated, Dict, List
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+from synapse_sdk.utils.pydantic.validators import non_blank
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class UploadRun(Run):
+    pass
+
+
+class UploadParams(BaseModel):
+    """Upload action parameters.
+
+    Args:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
+        checkpoint (int | None): The checkpoint of the action.
+        path (str): The path of the action.
+        storage (int): The storage of the action.
+        collection (int): The collection of the action.
+        project (int | None): The project of the action.
+        is_generate_tasks (bool): The flag to generate tasks.
+        is_generate_ground_truths (bool): The flag to generate ground truths
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None
+    path: str
+    storage: int
+    collection: int
+    project: int | None
+    is_generate_tasks: bool = False
+    is_generate_ground_truths: bool = False
+
+    @field_validator('storage', mode='before')
+    @classmethod
+    def check_storage_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend storage exists.
+
+        TODO: Need to define validation method naming convention.
+        TODO: Need to make validation method reusable.
+        """
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking storage exists.'))
+        return value
+
+    @field_validator('collection', mode='before')
+    @classmethod
+    def check_collection_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend collection exists."""
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_dataset(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking collection exists.'))
+        return value
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend project exists."""
+        if not value:
+            return value
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking project exists.'))
+        return value
+
+
+@register_action
+class UploadAction(Action):
+    """Upload action class.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+
+    Progress Categories:
+        analyze_collection: The progress category for the analyze collection process.
+        data_file_upload: The progress category for the upload process.
+        generate_data_units: The progress category for the generate data units process.
+        generate_tasks: The progress category for the generate tasks process.
+        generate_ground_truths: The progress category for the generate ground truths process.
+    """
+
+    name = 'upload'
+    category = PluginCategory.UPLOAD
+    method = RunMethod.JOB
+    progress_categories = {
+        'analyze_collection': {
+            'proportion': 5,
+        },
+        'upload_data_files': {
+            'proportion': 35,
+        },
+        'generate_data_units': {
+            'proportion': 20,
+        },
+        'generate_tasks': {
+            'proportion': 20,
+        },
+        'generate_ground_truths': {
+            'proportion': 20,
+        },
+    }
+
+    def get_uploader(self, path):
+        """Get uploader from entrypoint."""
+        return self.entrypoint(self.run, path)
+
+    def start(self) -> Dict:
+        """Start upload process.
+
+        Returns:
+            Dict: The result of the upload process.
+        """
+        # Setup path object with path and storage.
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+
+        # Initialize uploader.
+        uploader = self.get_uploader(pathlib_cwd)
+
+        # Analyze Collection file specifications to determine the data structure for upload.
+        self.run.set_progress(0, 1, category='analyze_collection')
+        file_specification_skeleton = self._analyze_collection()
+        self.run.set_progress(1, 1, category='analyze_collection')
+
+        # Setup result dict.
+        result = {}
+
+        # Organize data according to Collection file specification structure.
+        organized_files = uploader.handle_upload_files()
+        if not self._validate_organized_files(file_specification_skeleton, organized_files):
+            self.run.log_message('Validate organized files failed.')
+            return result
+
+        # Upload files to synapse-backend.
+        organized_files_count = len(organized_files)
+        if not organized_files_count:
+            self.run.log_message('Files not found on the path.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, organized_files_count, category='upload_data_files')
+        self.run.log_message('Uploading data files...')
+        result['uploaded_files'] = self._upload_files(organized_files)
+        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+        self.run.log_message('Upload data files completed.')
+
+        # Generate data units for the uploaded data.
+        upload_result_count = len(result['uploaded_files'])
+        if not upload_result_count:
+            self.run.log_message('No files were uploaded.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, upload_result_count, category='generate_data_units')
+        generated_data_units = self._generate_data_units(result['uploaded_files'])
+        result['generated_data_units'] = generated_data_units
+        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+
+        # Setup task with uploaded synapse-backend data units.
+        if not len(generated_data_units):
+            self.run.log_message('No data units were generated.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, 1, category='generate_tasks')
+        if self.config['options']['allow_generate_tasks'] and self.params['is_generate_tasks']:
+            self.run.log_message('Generating tasks with data files...')
+            self._generate_tasks(generated_data_units)
+            self.run.log_message('Generating tasks completed')
+        else:
+            self.run.log_message('Generating tasks process has passed.')
+
+        self.run.set_progress(1, 1, category='generate_tasks')
+
+        # Generate ground truths for the uploaded data.
+        # TODO: Need to add ground truths generation logic later.
+        self.run.set_progress(0, 1, category='generate_ground_truths')
+        if self.config['options']['allow_generate_ground_truths'] and self.params['is_generate_ground_truths']:
+            self.run.log_message('Generating ground truths...')
+            self._generate_ground_truths()
+            self.run.log_message('Generating ground truths completed')
+        else:
+            self.run.log_message('Generating ground truths process has passed.')
+        self.run.set_progress(1, 1, category='generate_ground_truths')
+
+        return result
+
+    def _analyze_collection(self) -> Dict:
+        """Analyze Synapse Collection Specifications.
+
+        Returns:
+            Dict: The file specifications of the collection.
+        """
+        client = self.run.client
+        collection_id = self.params['collection']
+        collection = client.get_dataset(collection_id)
+        return collection['file_specifications']
+
+    def _validate_organized_files(self, file_specification_skeleton: Dict, organized_files: List) -> bool:
+        """Validate organized files from Uploader."""
+        return True
+
+    def _upload_files(self, organized_files) -> List:
+        """Upload files to synapse-backend.
+
+        Returns:
+            Dict: The result of the upload.
+        """
+        client = self.run.client
+        collection_id = self.params['collection']
+        upload_result = []
+        organized_files_count = len(organized_files)
+        current_progress = 0
+        for organized_file in organized_files:
+            upload_result.append(client.upload_data_file(organized_file, collection_id))
+            self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
+            current_progress += 1
+        return upload_result
+
+    def _generate_data_units(self, uploaded_files: List) -> List:
+        """Generate data units for the uploaded data.
+
+        TODO: make batch size configurable.
+
+        Returns:
+            Dict: The result of the generate data units process.
+        """
+        client = self.run.client
+
+        generation_result = []
+        current_progress = 0
+        batches = get_batched_list(uploaded_files, 100)
+        batches_count = len(batches)
+        for batch in batches:
+            generation_result.append(client.create_data_units(batch))
+            self.run.set_progress(current_progress, batches_count, category='generate_data_units')
+            current_progress += 1
+        return generation_result
+
+    def _generate_tasks(self, generated_data_units: List):
+        """Setup task with uploaded synapse-backend data units.
+
+        TODO: make batch size configurable.
+        """
+
+        # Prepare batches for processing
+        client = self.run.client
+        project_id = self.params['project']
+        current_progress = 0
+
+        # Generate tasks
+        generated_data_units_count = len(generated_data_units)
+        for data_units in generated_data_units:
+            tasks_data = []
+            for data_unit in data_units:
+                task_data = {'project': project_id, 'data_unit': data_unit['id']}
+                tasks_data.append(task_data)
+
+            if tasks_data:
+                client.create_tasks(tasks_data)
+
+            self.run.set_progress(current_progress, generated_data_units_count, category='generate_tasks')
+            current_progress += 1
+
+    def _generate_ground_truths(self):
+        """Generate ground truths for the uploaded data.
+
+        TODO: Need to add ground truths generation logic later.
+        """

synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -0,0 +1,6 @@
+actions:
+  upload:
+    entrypoint: plugin.upload.Uploader
+    options:
+      allow_generate_tasks: false
+      allow_generate_ground_truths: false

synapse_sdk/plugins/categories/upload/templates/plugin/upload.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+from typing import List
+
+
+class Uploader:
+    """Plugin upload action class.
+
+    * Organize, upload, setup task, generate ground truths for the uploaded data.
+    """
+
+    def __init__(self, run, path: Path, *args, **kwargs):
+        """Initialize the plugin upload action class.
+
+        Args:
+            run: Plugin run object.
+            path: pathlib object by upload target destination path.
+        """
+        self.run = run
+        self.path = path
+
+    def handle_upload_files(self) -> List:
+        """Handle upload files.
+
+        * Organize data according to collection file specification structure.
+        * Structure files according to the file specification of the target collection.
+
+        Returns:
+            List: List of dictionaries containing 'files' and 'meta'.
+
+        Examples:
+            [
+              {
+          "files": {
+              'image_1': image_1_pathlib_object,
+              'image_2': image_2_pathlib_object,
+              'meta_1': meta_1_pathlib_object,
+          },
+          "meta": {
+            "key": "value"
+          }
+              }
+            ]
+        """
+        return []

synapse_sdk/plugins/enums.py

@@ -2,6 +2,8 @@ from enum import Enum
 
 
 class RunMethod(Enum):
+    """Plugin Execution Methods."""
+
     JOB = 'job'
     TASK = 'task'
     RESTAPI = 'restapi'
@@ -10,7 +12,7 @@ class RunMethod(Enum):
 class PluginCategory(Enum):
     NEURAL_NET = 'neural_net'
     EXPORT = 'export'
-    IMPORT = 'import'
+    UPLOAD = 'upload'
     SMART_TOOL = 'smart_tool'
     POST_ANNOTATION = 'post_annotation'
     PRE_ANNOTATION = 'pre_annotation'

synapse_sdk/plugins/models.py

@@ -87,6 +87,15 @@ class PluginRelease:
 
 
 class Run:
+    """Run class for manage plugin run istance.
+
+    Attrs:
+        job_id: plugin run job id
+        context: plugin run context
+        client: backend client for communicate with backend
+        logger: logger for log plugin run events
+    """
+
     logger = None
     job_id = None
     context = None
@@ -111,6 +120,13 @@ class Run:
             self.logger = ConsoleLogger(**kwargs)
 
     def set_progress(self, current, total, category=''):
+        """Set progress for plugin run.
+
+        Args:
+            current: current progress value
+            total: total progress value
+            category: progress category
+        """
         self.logger.set_progress(current, total, category)
 
     def log(self, event, data, file=None):

synapse_sdk/utils/storage/__init__.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from urllib.parse import urlparse
 
 from synapse_sdk.i18n import gettext as _
@@ -5,16 +6,33 @@ from synapse_sdk.utils.storage.registry import STORAGE_PROVIDERS
 
 
 def get_storage(connection_param: str | dict):
+    """Get storage class with connection param.
+
+    Args:
+        connection_param (str | dict): The connection param for the Storage provider.
+
+    Returns:
+        BaseStorage: The storage class object with connection param.
+    """
     storage_scheme = None
     if isinstance(connection_param, dict):
         storage_scheme = connection_param['provider']
     else:
         storage_scheme = urlparse(connection_param).scheme
 
-    assert storage_scheme in STORAGE_PROVIDERS.keys(), _('지원하지 않는 저장소입니다.')
+    assert storage_scheme in STORAGE_PROVIDERS.keys(), _('Storage provider not supported.')
     return STORAGE_PROVIDERS[storage_scheme](connection_param)
 
 
-def get_pathlib(storage_config, path_root):
+def get_pathlib(storage_config: str | dict, path_root: str) -> Path:
+    """Get pathlib object with synapse-backend storage config.
+
+    Args:
+        storage_config (str | dict): The storage config by synapse-backend storage api.
+        path_root (str): The path root.
+
+    Returns:
+        pathlib.Path: The pathlib object.
+    """
     storage_class = get_storage(storage_config)
     return storage_class.get_pathlib(path_root)

{synapse_sdk-1.0.0a31.dist-info → synapse_sdk-1.0.0a33.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a31
+Version: 1.0.0a33
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -21,6 +21,7 @@ Requires-Dist: universal-pathlib
 Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
+Dynamic: license-file
 
 This is the SDK to develop synapse plugins