synapse-sdk 1.0.0a30__tar.gz → 1.0.0a32__tar.gz

{synapse_sdk-1.0.0a30/synapse_sdk.egg-info → synapse_sdk-1.0.0a32}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: synapse-sdk
-Version: 1.0.0a30
+Version: 1.0.0a32
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -21,6 +21,7 @@ Requires-Dist: universal-pathlib
 Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
+Dynamic: license-file
 
 This is the SDK to develop synapse plugins
 

synapse_sdk-1.0.0a32/synapse_sdk/clients/backend/dataset.py

@@ -0,0 +1,102 @@
+from multiprocessing import Pool
+from pathlib import Path
+from typing import Dict, Optional
+
+from tqdm import tqdm
+
+from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.clients.utils import get_batched_list
+
+
+class DatasetClientMixin(BaseClient):
+    def list_dataset(self):
+        path = 'datasets/'
+        return self._list(path)
+
+    def get_dataset(self, dataset_id):
+        """Get dataset from synapse-backend.
+
+        Args:
+            dataset_id: The dataset id to get.
+        """
+        path = f'datasets/{dataset_id}/?expand=file_specifications'
+        return self._get(path)
+
+    def create_data_file(self, file_path: Path):
+        """Create data file to synapse-backend.
+
+        Args:
+            file_path: The file pathlib object to upload.
+        """
+        path = 'data_files/'
+        return self._post(path, files={'file': file_path})
+
+    def create_data_units(self, data):
+        """Create data units to synapse-backend.
+
+        Args:
+            data: The data bindings to upload from create_data_file interface.
+        """
+        path = 'data_units/'
+        return self._post(path, data=data)
+
+    def upload_dataset(
+        self,
+        dataset_id: int,
+        dataset: Dict,
+        project_id: Optional[int] = None,
+        batch_size: int = 1000,
+        process_pool: int = 10,
+    ):
+        """Upload dataset to synapse-backend.
+
+        Args:
+            dataset_id: The dataset id to upload the data to.
+            dataset: The dataset to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            project_id: The project id to upload the data to.
+            batch_size: The batch size to upload the data.
+            process_pool: The process pool to upload the data.
+        """
+        # TODO validate dataset with schema
+
+        params = [(data, dataset_id) for data in dataset]
+
+        with Pool(processes=process_pool) as pool:
+            dataset = pool.starmap(self.upload_data_file, tqdm(params))
+
+        batches = get_batched_list(dataset, batch_size)
+
+        for batch in tqdm(batches):
+            data_units = self.create_data_units(batch)
+
+            if project_id:
+                tasks_data = []
+                for data, data_unit in zip(batch, data_units):
+                    task_data = {'project': project_id, 'data_unit': data_unit['id']}
+                    # TODO: Additional logic needed here if task data storage is required during import.
+
+                    tasks_data.append(task_data)
+
+                self.create_tasks(tasks_data)
+
+    def upload_data_file(self, data: Dict, dataset_id: int) -> Dict:
+        """Upload files to synapse-backend.
+
+        Args:
+            data: The data to upload.
+                * structure:
+                    - files: The files to upload. (key: file name, value: file pathlib object)
+                    - meta: The meta data to upload.
+            dataset_id: The dataset id to upload the data to.
+
+        Returns:
+            Dict: The result of the upload.
+        """
+        for name, path in data['files'].items():
+            data_file = self.create_data_file(path)
+            data['dataset'] = dataset_id
+            data['files'][name] = {'checksum': data_file['checksum'], 'path': str(path)}
+        return data

{synapse_sdk-1.0.0a30 → synapse_sdk-1.0.0a32}/synapse_sdk/clients/base.py

@@ -40,37 +40,31 @@ class BaseClient:
         headers = self._get_headers()
         headers.update(kwargs.pop('headers', {}))
 
+        # List to store opened files to close after request
+        opened_files = []
+
         if method in ['post', 'put', 'patch']:
             # If files are included in the request, open them as binary files
             if kwargs.get('files') is not None:
-                opened_files = []  # List to store opened files
-                try:
-                    for name, file in kwargs['files'].items():
-                        # If file is a path string, bind it as a Path object and open
-                        if isinstance(file, str):
-                            opened_file = open(Path(file), mode='rb')
-                            kwargs['files'][name] = opened_file
-                            opened_files.append(opened_file)
-                        # If file is a Path object, open it directly
-                        elif isinstance(file, Path):
-                            opened_file = open(file, mode='rb')
-                            kwargs['files'][name] = opened_file
-                            opened_files.append(opened_file)
-                    if 'data' in kwargs:
-                        for name, value in kwargs['data'].items():
-                            if isinstance(value, dict):
-                                kwargs['data'][name] = json.dumps(value)
-                finally:
-                    # Close all opened files
-                    for opened_file in opened_files:
-                        opened_file.close()
-
+                for name, file in kwargs['files'].items():
+                    # Handle both string and Path object cases
+                    if isinstance(file, str):
+                        file = Path(file)
+                    if isinstance(file, Path):
+                        opened_file = file.open(mode='rb')
+                        kwargs['files'][name] = (file.name, opened_file)
+                        opened_files.append(opened_file)
+                if 'data' in kwargs:
+                    for name, value in kwargs['data'].items():
+                        if isinstance(value, dict):
+                            kwargs['data'][name] = json.dumps(value)
             else:
                 headers['Content-Type'] = 'application/json'
                 if 'data' in kwargs:
                     kwargs['data'] = json.dumps(kwargs['data'])
 
         try:
+            # Send request
             response = getattr(self.requests_session, method)(url, headers=headers, **kwargs)
             if not response.ok:
                 raise ClientError(
@@ -79,6 +73,10 @@ class BaseClient:
         except requests.ConnectionError:
             raise ClientError(408, f'{self.name} is not responding')
 
+        # Close all opened files
+        for opened_file in opened_files:
+            opened_file.close()
+
         return self._post_response(response)
 
     def _post_response(self, response):

{synapse_sdk-1.0.0a30 → synapse_sdk-1.0.0a32}/synapse_sdk/plugins/categories/base.py

@@ -17,6 +17,30 @@ from synapse_sdk.utils.pydantic.errors import pydantic_to_drf_error
 
 
 class Action:
+    """Base class for all plugin actions.
+
+    Attrs:
+        name (str): The name of the action.
+        category (PluginCategory): The category of the action.
+        method (RunMethod): The method to run of the action.
+        run_class (Run): The class to run the action.
+        params_model (BaseModel): The model to validate the params.
+        progress_categories (List[str]): The categories to update the progress.
+        params (Dict): The params to run the action.
+        plugin_config (Dict): The plugin config.
+        plugin_release (PluginRelease): The plugin release.
+        config (Dict): The action config.
+        requirements (List[str]): The requirements to install.
+        job_id (str): The job id.
+        direct (bool): The flag to run the action directly.
+        debug (bool): The flag to run the action in debug mode.
+        envs (Dict): The runtime envs.
+        run (Run): The run instance.
+
+    Raises:
+        ActionError: If the action fails.
+    """
+
     # class 변수
     name = None
     category = None
@@ -159,11 +183,19 @@ class Action:
         return getattr(self, f'start_by_{self.method.value}')()
 
     def start(self):
+        """Start the action.
+
+        TODO: Specify the return type of start method for overrided methods.
+        """
         if self.method == RunMethod.JOB:
             return self.entrypoint(self.run, **self.params)
         return self.entrypoint(**self.params)
 
     def start_by_task(self):
+        """Ray Task based execution.
+
+        * A task method that simply executes the entrypoint without job management functionality.
+        """
         import ray
         from ray.exceptions import RayTaskError
 
@@ -195,6 +227,10 @@ class Action:
             raise ActionError(e.cause)
 
     def start_by_job(self):
+        """Ray Job based execution.
+
+        * Executes the entrypoint with Ray job. Ray job manages the entrypoint execution and stores the results.
+        """
         main_options = []
         options = ['run', '--direct']
         arguments = [self.name, f'{json.dumps(json.dumps(self.params))}']
@@ -215,6 +251,10 @@ class Action:
         )
 
     def start_by_restapi(self):
+        """Ray Serve based execution.
+
+        * This method executes a Fastapi endpoint defined within the Plugin.
+        """
         path = self.params.pop('path', '')
         method = self.params.pop('method')
 

{synapse_sdk-1.0.0a30 → synapse_sdk-1.0.0a32}/synapse_sdk/plugins/categories/export/actions/export.py

@@ -61,11 +61,8 @@ class ExportAction(Action):
             }
 
     def get_filtered_results(self):
-        """Get filtered ground truth."""
-        ground_truth_dataset_version = self.params['ground_truth_dataset_version']
-        self.params['filter']['ground_truth_dataset'] = self.client.get_ground_truth_version(
-            ground_truth_dataset_version
-        )['ground_truth_dataset']
+        """Get filtered ground truth events."""
+        self.params['filter']['ground_truth_dataset_versions'] = self.params['ground_truth_dataset_version']
         filters = {'expand': 'data', **self.params['filter']}
 
         try:

{synapse_sdk-1.0.0a30 → synapse_sdk-1.0.0a32}/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -1,9 +1,8 @@
 import json
+from pathlib import Path
 
 import requests
 
-from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
-
 
 def export(run, input_dataset, path_root, **params):
     """Executes the export task.
@@ -14,7 +13,7 @@ def export(run, input_dataset, path_root, **params):
             - data (dict): dm_schema_data information.
             - files (dict): File information. Includes file URL, original file path, metadata, etc.
             - id (int): ground_truth ID
-        path_root : Save path
+        path_root : pathlib object, the path to export
         **params: Additional parameters
 
     Returns:
@@ -24,10 +23,11 @@ def export(run, input_dataset, path_root, **params):
     path_root.mkdir(parents=True, exist_ok=True)
     run.log_message('Starting export process.')
 
-    # results: Contains all information fetched through the list API.
-    results = params.get('results', [])
+    # results contains all information fetched through the list API.
+    # example:
+    #   params.get('results', [])
 
-    save_original_file = params.get('save_original_file')
+    save_original_file_flag = params.get('save_original_file')
     errors_json_file_list = []
     errors_original_file_list = []
 
@@ -39,21 +39,28 @@ def export(run, input_dataset, path_root, **params):
     origin_files_output_path = path_root / 'origin_files'
     origin_files_output_path.mkdir(parents=True, exist_ok=True)
 
-    total = len(results)
-    for no, input_data in enumerate(input_dataset):
+    total = params['count']
+    # progress init
+    run.set_progress(0, total, category='dataset_conversion')
+    for no, input_data in enumerate(input_dataset, start=1):
         run.set_progress(no, total, category='dataset_conversion')
+        if no == 1:
+            run.log_message('Converting dataset.')
         preprocessed_data = before_convert(input_data)
         converted_data = convert_data(preprocessed_data)
         final_data = after_convert(converted_data)
 
         # Call if original file extraction is needed
-        if save_original_file:
+        if save_original_file_flag:
+            if no == 1:
+                run.log_message('Saving original file.')
             save_original_file(final_data, origin_files_output_path, errors_original_file_list)
 
         # Extract data as JSON files
+        if no == 1:
+            run.log_message('Saving json file.')
         save_as_json(final_data, json_output_path, errors_json_file_list)
 
-    run.log_message('Saving converted dataset.')
     run.end_log()
 
     # Save error list files
@@ -62,7 +69,7 @@ def export(run, input_dataset, path_root, **params):
         with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
             json.dump(export_error_file, f, indent=4, ensure_ascii=False)
 
-    return {'export_path': path_root}
+    return {'export_path': str(path_root)}
 
 
 def convert_data(data):
@@ -80,19 +87,29 @@ def after_convert(data):
     return data
 
 
-def save_original_file(result, base_path, error_file_list):
-    """Saves the original file.
+def get_original_file_pathlib(files):
+    """Retrieve the original file path from the given file information.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
+        files (dict): A dictionary containing file information, including file URL,
+                      original file path, metadata, etc.
 
     Returns:
-        base_path (str): Save path
+        pathlib.Path: The original file path extracted from the metadata.
+    """
+    return Path(next(iter(files.values()))['meta']['path_original'])
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
     file_url = next(iter(result['files'].values()))['url']
-    file_name = get_original_file_path(result['files']).name
+    file_name = get_original_file_pathlib(result['files']).name
     response = requests.get(file_url)
     try:
         with (base_path / file_name).open('wb') as file:
@@ -100,27 +117,20 @@ def save_original_file(result, base_path, error_file_list):
     except Exception as e:
         error_file_list.append([file_name, str(e)])
 
-    return base_path
-
 
 def save_as_json(result, base_path, error_file_list):
     """Saves the data as a JSON file.
 
     Args:
-        result (dict): Result data
-        base_path (Path): Save path
-        error_file_list (list): List of error files
-
-    Returns:
-        base_path (str): Save path
+        result (dict): API response data containing file information.
+        base_path (Path): The directory where the file will be saved.
+        error_file_list (list): A list to store error files.
     """
     # Default save file name: original file name
-    file_name = get_original_file_path(result['files']).stem
+    file_name = get_original_file_pathlib(result['files']).stem
     json_data = result['data']
     try:
         with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
             json.dump(json_data, f, indent=4, ensure_ascii=False)
     except Exception as e:
         error_file_list.append([f'{file_name}.json', str(e)])
-
-    return base_path

{synapse_sdk-1.0.0a30 → synapse_sdk-1.0.0a32}/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -51,7 +51,7 @@ class TrainParams(BaseModel):
                 params={
                     'ids_ex': action.job_id,
                     'category': 'neural_net',
-                    'action': 'train',
+                    'job__action': 'train',
                     'is_active': True,
                     'params': f'name:{value}',
                 },