synapse-sdk 1.0.0a31__tar.gz → 1.0.0a32__tar.gz

{synapse_sdk-1.0.0a31/synapse_sdk.egg-info → synapse_sdk-1.0.0a32}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a31
+Version: 1.0.0a32
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -21,6 +21,7 @@ Requires-Dist: universal-pathlib
 Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
+Dynamic: license-file
 
 This is the SDK to develop synapse plugins
 

synapse_sdk-1.0.0a32/synapse_sdk/clients/backend/dataset.py

@@ -0,0 +1,102 @@
+from multiprocessing import Pool
+from pathlib import Path
+from typing import Dict, Optional
+
+from tqdm import tqdm
+
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_batched_list
+
+
+class DatasetClientMixin(BaseClient):
+    def list_dataset(self):
+        path = 'datasets/'
+        return self._list(path)
+
+    def get_dataset(self, dataset_id):
+        """Get dataset from synapse-backend.
+
+        Args:
+            dataset_id: The dataset id to get.
+        """
+        path = f'datasets/{dataset_id}/?expand=file_specifications'
+        return self._get(path)
+
+    def create_data_file(self, file_path: Path):
+        """Create data file to synapse-backend.
+
+        Args:
+            file_path: The file pathlib object to upload.
+        """
+        path = 'data_files/'
+        return self._post(path, files={'file': file_path})
+
+    def create_data_units(self, data):
+        """Create data units to synapse-backend.
+
+        Args:
+            data: The data bindings to upload from create_data_file interface.
+        """
+        path = 'data_units/'
+        return self._post(path, data=data)
+
+    def upload_dataset(
+        self,
+        dataset_id: int,
+        dataset: Dict,
+        project_id: Optional[int] = None,
+        batch_size: int = 1000,
+        process_pool: int = 10,
+    ):
+        """Upload dataset to synapse-backend.
+
+        Args:
+            dataset_id: The dataset id to upload the data to.
+            dataset: The dataset to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            project_id: The project id to upload the data to.
+            batch_size: The batch size to upload the data.
+            process_pool: The process pool to upload the data.
+        """
+        # TODO validate dataset with schema
+
+        params = [(data, dataset_id) for data in dataset]
+
+        with Pool(processes=process_pool) as pool:
+            dataset = pool.starmap(self.upload_data_file, tqdm(params))
+
+        batches = get_batched_list(dataset, batch_size)
+
+        for batch in tqdm(batches):
+            data_units = self.create_data_units(batch)
+
+            if project_id:
+                tasks_data = []
+                for data, data_unit in zip(batch, data_units):
+                    task_data = {'project': project_id, 'data_unit': data_unit['id']}
+                    # TODO: Additional logic needed here if task data storage is required during import.
+
+                    tasks_data.append(task_data)
+
+                self.create_tasks(tasks_data)
+
+    def upload_data_file(self, data: Dict, dataset_id: int) -> Dict:
+        """Upload files to synapse-backend.
+
+        Args:
+            data: The data to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            dataset_id: The dataset id to upload the data to.
+
+        Returns:
+            Dict: The result of the upload.
+        """
+        for name, path in data['files'].items():
+            data_file = self.create_data_file(path)
+            data['dataset'] = dataset_id
+            data['files'][name] = {'checksum': data_file['checksum'], 'path': str(path)}
+        return data

{synapse_sdk-1.0.0a31 → synapse_sdk-1.0.0a32}/synapse_sdk/clients/base.py

@@ -47,15 +47,12 @@ class BaseClient:
             # If files are included in the request, open them as binary files
             if kwargs.get('files') is not None:
                 for name, file in kwargs['files'].items():
-                    # If file is a path string, bind it as a Path object and open
+                    # Handle both string and Path object cases
                     if isinstance(file, str):
-                        opened_file = Path(file).open(mode='rb')
-                        kwargs['files'][name] = opened_file
-                        opened_files.append(opened_file)
-                    # If file is a Path object, open it directly
-                    elif isinstance(file, Path):
+                        file = Path(file)
+                    if isinstance(file, Path):
                         opened_file = file.open(mode='rb')
-                        kwargs['files'][name] = opened_file
+                        kwargs['files'][name] = (file.name, opened_file)
                         opened_files.append(opened_file)
                 if 'data' in kwargs:
                     for name, value in kwargs['data'].items():
@@ -67,6 +64,7 @@ class BaseClient:
                     kwargs['data'] = json.dumps(kwargs['data'])
 
         try:
+            # Send request
             response = getattr(self.requests_session, method)(url, headers=headers, **kwargs)
             if not response.ok:
                 raise ClientError(

{synapse_sdk-1.0.0a31 → synapse_sdk-1.0.0a32}/synapse_sdk/plugins/categories/base.py

@@ -17,6 +17,30 @@ from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
 
 class Action:
+    """Base class for all plugin actions.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+        run_class (Run): The class to run the action.
+        params_model (BaseModel): The model to validate the params.
+        progress_categories (List[str]): The categories to update the progress.
+        params (Dict): The params to run the action.
+        plugin_config (Dict): The plugin config.
+        plugin_release (PluginRelease): The plugin release.
+        config (Dict): The action config.
+        requirements (List[str]): The requirements to install.
+        job_id (str): The job id.
+        direct (bool): The flag to run the action directly.
+        debug (bool): The flag to run the action in debug mode.
+        envs (Dict): The runtime envs.
+        run (Run): The run instance.
+
+    Raises:
+        ActionError: If the action fails.
+    """
+
     # class 변수
     name = None
     category = None
@@ -159,11 +183,19 @@ class Action:
         return getattr(self, f'start_by_{self.method.value}')()
 
     def start(self):
+        """Start the action.
+
+        TODO: Specify the return type of start method for overrided methods.
+        """
         if self.method == RunMethod.JOB:
             return self.entrypoint(self.run, **self.params)
         return self.entrypoint(**self.params)
 
     def start_by_task(self):
+        """Ray Task based execution.
+
+        * A task method that simply executes the entrypoint without job management functionality.
+        """
         import ray
         from ray.exceptions import RayTaskError
 
@@ -195,6 +227,10 @@ class Action:
             raise ActionError(e.cause)
 
     def start_by_job(self):
+        """Ray Job based execution.
+
+        * Executes the entrypoint with Ray job. Ray job manages the entrypoint execution and stores the results.
+        """
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -215,6 +251,10 @@ class Action:
         )
 
     def start_by_restapi(self):
+        """Ray Serve based execution.
+
+        * This method executes a Fastapi endpoint defined within the Plugin.
+        """
         path = self.params.pop('path', '')
         method = self.params.pop('method')
 

{synapse_sdk-1.0.0a31 → synapse_sdk-1.0.0a32}/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -1,9 +1,8 @@
 import json
+from pathlib import Path
 
 import requests
 
-from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
-
 
 def export(run, input_dataset, path_root, **params):
     """Executes the export task.
@@ -14,7 +13,7 @@ def export(run, input_dataset, path_root, **params):
             - data (dict): dm_schema_data information.
             - files (dict): File information. Includes file URL, original file path, metadata, etc.
             - id (int): ground_truth ID
-        path_root : Save path
+        path_root : pathlib object, the path to export
         **params: Additional parameters
 
     Returns:
@@ -24,10 +23,11 @@ def export(run, input_dataset, path_root, **params):
     path_root.mkdir(parents=True, exist_ok=True)
     run.log_message('Starting export process.')
 
-    # results: Contains all information fetched through the list API.
-    results = params.get('results', [])
+    # results contains all information fetched through the list API.
+    # example:
+    #   params.get('results', [])
 
-    save_original_file = params.get('save_original_file')
+    save_original_file_flag = params.get('save_original_file')
     errors_json_file_list = []
     errors_original_file_list = []
 
@@ -39,21 +39,28 @@ def export(run, input_dataset, path_root, **params):
     origin_files_output_path = path_root / 'origin_files'
     origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
-    total = len(results)
-    for no, input_data in enumerate(input_dataset):
+    total = params['count']
+    # progress init
+    run.set_progress(0, total, category='dataset_conversion')
+    for no, input_data in enumerate(input_dataset, start=1):
         run.set_progress(no, total, category='dataset_conversion')
+        if no == 1:
+            run.log_message('Converting dataset.')
         preprocessed_data = before_convert(input_data)
         converted_data = convert_data(preprocessed_data)
         final_data = after_convert(converted_data)
 
         # Call if original file extraction is needed
-        if save_original_file:
+        if save_original_file_flag:
+            if no == 1:
+                run.log_message('Saving original file.')
             save_original_file(final_data, origin_files_output_path, errors_original_file_list)
 
         # Extract data as JSON files
+        if no == 1:
+            run.log_message('Saving json file.')
         save_as_json(final_data, json_output_path, errors_json_file_list)
 
-    run.log_message('Saving converted dataset.')
     run.end_log()
 
     # Save error list files
@@ -62,7 +69,7 @@ def export(run, input_dataset, path_root, **params):
         with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
             json.dump(export_error_file, f, indent=4, ensure_ascii=False)
 
-    return {'export_path': path_root}
+    return {'export_path': str(path_root)}
 
 
 def convert_data(data):
@@ -80,19 +87,29 @@ def after_convert(data):
     return data
 
 
-def save_original_file(result, base_path, error_file_list):
-    """Saves the original file.
+def get_original_file_pathlib(files):
+    """Retrieve the original file path from the given file information.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
+        files (dict): A dictionary containing file information, including file URL,
+                      original file path, metadata, etc.
 
     Returns:
-        base_path (str): Save path
+        pathlib.Path: The original file path extracted from the metadata.
+    """
+    return Path(next(iter(files.values()))['meta']['path_original'])
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
     file_url = next(iter(result['files'].values()))['url']
-    file_name = get_original_file_path(result['files']).name
+    file_name = get_original_file_pathlib(result['files']).name
     response = requests.get(file_url)
     try:
         with (base_path / file_name).open('wb') as file:
@@ -100,27 +117,20 @@ def save_original_file(result, base_path, error_file_list):
     except Exception as e:
         error_file_list.append([file_name, str(e)])
 
-    return base_path
-
 
 def save_as_json(result, base_path, error_file_list):
     """Saves the data as a JSON file.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
-
-    Returns:
-        base_path (str): Save path
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
     # Default save file name: original file name
-    file_name = get_original_file_path(result['files']).stem
+    file_name = get_original_file_pathlib(result['files']).stem
     json_data = result['data']
     try:
         with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
             json.dump(json_data, f, indent=4, ensure_ascii=False)
     except Exception as e:
         error_file_list.append([f'{file_name}.json', str(e)])
-
-    return base_path

synapse_sdk-1.0.0a32/synapse_sdk/plugins/categories/upload/actions/upload.py

@@ -0,0 +1,292 @@
+from enum import Enum
+from typing import Annotated, Dict, List
+
+from pydantic import AfterValidator, BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.clients.utils import get_batched_list
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.plugins.models import Run
+from synapse_sdk.shared.enums import Context
+from synapse_sdk.utils.pydantic.validators import non_blank
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class UploadRun(Run):
+    pass
+
+
+class UploadParams(BaseModel):
+    """Upload action parameters.
+
+    Args:
+        name (str): The name of the action.
+        description (str | None): The description of the action.
+        checkpoint (int | None): The checkpoint of the action.
+        path (str): The path of the action.
+        storage (int): The storage of the action.
+        collection (int): The collection of the action.
+        project (int | None): The project of the action.
+        is_generate_tasks (bool): The flag to generate tasks.
+        is_generate_ground_truths (bool): The flag to generate ground truths
+    """
+
+    name: Annotated[str, AfterValidator(non_blank)]
+    description: str | None
+    path: str
+    storage: int
+    collection: int
+    project: int | None
+    is_generate_tasks: bool = False
+    is_generate_ground_truths: bool = False
+
+    @field_validator('storage', mode='before')
+    @classmethod
+    def check_storage_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend storage exists.
+
+        TODO: Need to define validation method naming convention.
+        TODO: Need to make validation method reusable.
+        """
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking storage exists.'))
+        return value
+
+    @field_validator('collection', mode='before')
+    @classmethod
+    def check_collection_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend collection exists."""
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_dataset(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking collection exists.'))
+        return value
+
+    @field_validator('project', mode='before')
+    @classmethod
+    def check_project_exists(cls, value: str, info) -> str:
+        """Validate synapse-backend project exists."""
+        if not value:
+            return value
+
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_project(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Error occurred while checking project exists.'))
+        return value
+
+
+@register_action
+class UploadAction(Action):
+    """Upload action class.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+
+    Progress Categories:
+        analyze_collection: The progress category for the analyze collection process.
+        data_file_upload: The progress category for the upload process.
+        generate_data_units: The progress category for the generate data units process.
+        generate_tasks: The progress category for the generate tasks process.
+        generate_ground_truths: The progress category for the generate ground truths process.
+    """
+
+    name = 'upload'
+    category = PluginCategory.UPLOAD
+    method = RunMethod.JOB
+    progress_categories = {
+        'analyze_collection': {
+            'proportion': 5,
+        },
+        'upload_data_files': {
+            'proportion': 35,
+        },
+        'generate_data_units': {
+            'proportion': 20,
+        },
+        'generate_tasks': {
+            'proportion': 20,
+        },
+        'generate_ground_truths': {
+            'proportion': 20,
+        },
+    }
+
+    def get_uploader(self, path):
+        """Get uploader from entrypoint."""
+        return self.entrypoint(self.run, path)
+
+    def start(self) -> Dict:
+        """Start upload process.
+
+        Returns:
+            Dict: The result of the upload process.
+        """
+        # Setup path object with path and storage.
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+
+        # Initialize uploader.
+        uploader = self.get_uploader(pathlib_cwd)
+
+        # Analyze Collection file specifications to determine the data structure for upload.
+        self.run.set_progress(0, 1, category='analyze_collection')
+        file_specification_skeleton = self._analyze_collection()
+        self.run.set_progress(1, 1, category='analyze_collection')
+
+        # Setup result dict.
+        result = {}
+
+        # Organize data according to Collection file specification structure.
+        organized_files = uploader.handle_upload_files()
+        if not self._validate_organized_files(file_specification_skeleton, organized_files):
+            self.run.log_message('Validate organized files failed.')
+            return result
+
+        # Upload files to synapse-backend.
+        organized_files_count = len(organized_files)
+        if not organized_files_count:
+            self.run.log_message('Files not found on the path.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, organized_files_count, category='upload_data_files')
+        self.run.log_message('Uploading data files...')
+        result['uploaded_files'] = self._upload_files(organized_files)
+        self.run.set_progress(organized_files_count, organized_files_count, category='upload_data_files')
+        self.run.log_message('Upload data files completed.')
+
+        # Generate data units for the uploaded data.
+        upload_result_count = len(result['uploaded_files'])
+        if not upload_result_count:
+            self.run.log_message('No files were uploaded.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, upload_result_count, category='generate_data_units')
+        generated_data_units = self._generate_data_units(result['uploaded_files'])
+        result['generated_data_units'] = generated_data_units
+        self.run.set_progress(upload_result_count, upload_result_count, category='generate_data_units')
+
+        # Setup task with uploaded synapse-backend data units.
+        if not len(generated_data_units):
+            self.run.log_message('No data units were generated.', context=Context.WARNING.value)
+            return result
+
+        self.run.set_progress(0, 1, category='generate_tasks')
+        if self.config['options']['allow_generate_tasks'] and self.params['is_generate_tasks']:
+            self.run.log_message('Generating tasks with data files...')
+            self._generate_tasks(generated_data_units)
+            self.run.log_message('Generating tasks completed')
+        else:
+            self.run.log_message('Generating tasks process has passed.')
+
+        self.run.set_progress(1, 1, category='generate_tasks')
+
+        # Generate ground truths for the uploaded data.
+        # TODO: Need to add ground truths generation logic later.
+        self.run.set_progress(0, 1, category='generate_ground_truths')
+        if self.config['options']['allow_generate_ground_truths'] and self.params['is_generate_ground_truths']:
+            self.run.log_message('Generating ground truths...')
+            self._generate_ground_truths()
+            self.run.log_message('Generating ground truths completed')
+        else:
+            self.run.log_message('Generating ground truths process has passed.')
+        self.run.set_progress(1, 1, category='generate_ground_truths')
+
+        return result
+
+    def _analyze_collection(self) -> Dict:
+        """Analyze Synapse Collection Specifications.
+
+        Returns:
+            Dict: The file specifications of the collection.
+        """
+        client = self.run.client
+        collection_id = self.params['collection']
+        collection = client.get_dataset(collection_id)
+        return collection['file_specifications']
+
+    def _validate_organized_files(self, file_specification_skeleton: Dict, organized_files: List) -> bool:
+        """Validate organized files from Uploader."""
+        return True
+
+    def _upload_files(self, organized_files) -> List:
+        """Upload files to synapse-backend.
+
+        Returns:
+            Dict: The result of the upload.
+        """
+        client = self.run.client
+        collection_id = self.params['collection']
+        upload_result = []
+        organized_files_count = len(organized_files)
+        current_progress = 0
+        for organized_file in organized_files:
+            upload_result.append(client.upload_data_file(organized_file, collection_id))
+            self.run.set_progress(current_progress, organized_files_count, category='upload_data_files')
+            current_progress += 1
+        return upload_result
+
+    def _generate_data_units(self, uploaded_files: List) -> List:
+        """Generate data units for the uploaded data.
+
+        TODO: make batch size configurable.
+
+        Returns:
+            Dict: The result of the generate data units process.
+        """
+        client = self.run.client
+
+        generation_result = []
+        current_progress = 0
+        batches = get_batched_list(uploaded_files, 100)
+        batches_count = len(batches)
+        for batch in batches:
+            generation_result.append(client.create_data_units(batch))
+            self.run.set_progress(current_progress, batches_count, category='generate_data_units')
+            current_progress += 1
+        return generation_result
+
+    def _generate_tasks(self, generated_data_units: List):
+        """Setup task with uploaded synapse-backend data units.
+
+        TODO: make batch size configurable.
+        """
+
+        # Prepare batches for processing
+        client = self.run.client
+        project_id = self.params['project']
+        current_progress = 0
+
+        # Generate tasks
+        generated_data_units_count = len(generated_data_units)
+        for data_units in generated_data_units:
+            tasks_data = []
+            for data_unit in data_units:
+                task_data = {'project': project_id, 'data_unit': data_unit['id']}
+                tasks_data.append(task_data)
+
+            if tasks_data:
+                client.create_tasks(tasks_data)
+
+            self.run.set_progress(current_progress, generated_data_units_count, category='generate_tasks')
+            current_progress += 1
+
+    def _generate_ground_truths(self):
+        """Generate ground truths for the uploaded data.
+
+        TODO: Need to add ground truths generation logic later.
+        """

synapse_sdk-1.0.0a32/synapse_sdk/plugins/categories/upload/templates/config.yaml

@@ -0,0 +1,6 @@
+actions:
+  upload:
+    entrypoint: plugin.upload.Uploader
+    options:
+      allow_generate_tasks: false
+      allow_generate_ground_truths: false