synapse-sdk 1.0.0a28__tar.gz → 1.0.0a30__tar.gz

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/.gitignore

@@ -7,6 +7,9 @@
 .idea
 *.iml
 
+# vscode
+.vscode
+
 # http
 /http/
 

{synapse_sdk-1.0.0a28/synapse_sdk.egg-info → synapse_sdk-1.0.0a30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: synapse-sdk
-Version: 1.0.0a28
+Version: 1.0.0a30
 Summary: synapse sdk
 Author-email: datamaker <developer@datamaker.io>
 License: MIT
@@ -17,6 +17,8 @@ Requires-Dist: python-dotenv
 Requires-Dist: pyyaml
 Requires-Dist: pydantic
 Requires-Dist: pyjwt
+Requires-Dist: universal-pathlib
+Requires-Dist: fsspec[gcs,s3,sftp]
 Provides-Extra: all
 Requires-Dist: ray[all]; extra == "all"
 

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/pyproject.toml

@@ -21,7 +21,9 @@ dependencies = [
     "python-dotenv",
     "pyyaml",
     "pydantic",
-    "pyjwt"
+    "pyjwt",
+    "universal-pathlib",
+    "fsspec[gcs,s3,sftp]"
 ]
 dynamic = ["version"]
 

synapse_sdk-1.0.0a30/requirements.test.txt

@@ -0,0 +1 @@
+pytest

synapse_sdk-1.0.0a30/requirements.txt

@@ -0,0 +1,11 @@
+click
+cookiecutter
+requests
+tqdm
+pyyaml
+pydantic
+pyjwt
+python-dotenv
+ray[all]>=2.43.0
+fsspec[gcs,s3,sftp]==2025.2.0
+universal-pathlib==0.2.6

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/cli/plugin/__init__.py

@@ -1,4 +1,5 @@
 import os
+import sys
 
 import click
 from dotenv import load_dotenv
@@ -12,6 +13,7 @@ from .run import run
 
 load_dotenv_default_alias()
 load_dotenv(os.path.join(os.getcwd(), '.env'), override=True)
+sys.path.append(os.getcwd())
 
 
 @click.group(context_settings={'obj': {}, 'auto_envvar_prefix': 'SYNAPSE_PLUGIN'})

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/clients/backend/integration.py

@@ -1,4 +1,5 @@
 from synapse_sdk.clients.base import BaseClient
+from synapse_sdk.utils.file import convert_file_to_base64
 
 
 class IntegrationClientMixin(BaseClient):
@@ -60,6 +61,13 @@ class IntegrationClientMixin(BaseClient):
 
     def create_logs(self, data):
         path = 'logs/'
+        if not isinstance(data, list):
+            data = [data]
+
+        for item in data:
+            if 'file' in item:
+                item['file'] = convert_file_to_base64(item['file'])
+
         return self._post(path, data=data)
 
     def create_serve_application(self, data):
@@ -69,3 +77,7 @@ class IntegrationClientMixin(BaseClient):
     def list_serve_applications(self, params=None, list_all=False):
         path = 'serve_applications/'
         return self._list(path, params=params, list_all=list_all)
+
+    def get_storage(self, pk):
+        path = f'storages/{pk}/'
+        return self._get(path)

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/clients/backend/ml.py

@@ -22,3 +22,7 @@ class MLClientMixin(BaseClient):
         path = 'ground_truth_events/'
         url_conversion = get_default_url_conversion(url_conversion, files_fields=['files'])
         return self._list(path, params=params, url_conversion=url_conversion, list_all=list_all)
+
+    def get_ground_truth_version(self, pk):
+        path = f'ground_truth_dataset_versions/{pk}/'
+        return self._get(path)

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/clients/base.py

@@ -25,20 +25,46 @@ class BaseClient:
     def _get_headers(self):
         return {}
 
-    def _request(self, method, path, **kwargs):
+    def _request(self, method: str, path: str, **kwargs) -> dict | str:
+        """Request handler for all HTTP methods.
+
+        Args:
+            method (str): HTTP method to use.
+            path (str): URL path to request.
+            **kwargs: Additional keyword arguments to pass to the request.
+
+        Returns:
+            dict | str: JSON response or text response.
+        """
         url = self._get_url(path)
         headers = self._get_headers()
         headers.update(kwargs.pop('headers', {}))
 
         if method in ['post', 'put', 'patch']:
+            # If files are included in the request, open them as binary files
             if kwargs.get('files') is not None:
-                for name, file in kwargs['files'].items():
-                    if isinstance(file, (str, Path)):
-                        kwargs['files'][name] = Path(str(file)).open(mode='rb')
-                if 'data' in kwargs:
-                    for name, value in kwargs['data'].items():
-                        if isinstance(value, dict):
-                            kwargs['data'][name] = json.dumps(value)
+                opened_files = []  # List to store opened files
+                try:
+                    for name, file in kwargs['files'].items():
+                        # If file is a path string, bind it as a Path object and open
+                        if isinstance(file, str):
+                            opened_file = open(Path(file), mode='rb')
+                            kwargs['files'][name] = opened_file
+                            opened_files.append(opened_file)
+                        # If file is a Path object, open it directly
+                        elif isinstance(file, Path):
+                            opened_file = open(file, mode='rb')
+                            kwargs['files'][name] = opened_file
+                            opened_files.append(opened_file)
+                    if 'data' in kwargs:
+                        for name, value in kwargs['data'].items():
+                            if isinstance(value, dict):
+                                kwargs['data'][name] = json.dumps(value)
+                finally:
+                    # Close all opened files
+                    for opened_file in opened_files:
+                        opened_file.close()
+
             else:
                 headers['Content-Type'] = 'application/json'
                 if 'data' in kwargs:

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/loggers.py

@@ -68,13 +68,16 @@ class BaseLogger:
 
         return progress
 
+    def log(self, action, data, file=None):
+        raise NotImplementedError
+
 
 class ConsoleLogger(BaseLogger):
     def set_progress(self, current, total, category=None):
         super().set_progress(current, total, category=category)
         print(self.get_current_progress())
 
-    def log(self, action, data):
+    def log(self, action, data, file=None):
         print(action, data)
 
 
@@ -99,7 +102,7 @@ class BackendLogger(BaseLogger):
         except ClientError:
             pass
 
-    def log(self, event, data):
+    def log(self, event, data, file=None):
         print(event, data)
 
         log = {
@@ -108,7 +111,11 @@ class BackendLogger(BaseLogger):
             'datetime': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
             'job': self.job_id,
         }
+        if file:
+            log['file'] = file
+
         self.logs_queue.append(log)
+
         try:
             self.client.create_logs(self.logs_queue)
             self.logs_queue.clear()

synapse_sdk-1.0.0a30/synapse_sdk/plugins/categories/export/actions/export.py

@@ -0,0 +1,85 @@
+from pydantic import BaseModel, field_validator
+from pydantic_core import PydanticCustomError
+
+from synapse_sdk.clients.exceptions import ClientError
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.plugins.categories.base import Action
+from synapse_sdk.plugins.categories.decorators import register_action
+from synapse_sdk.plugins.enums import PluginCategory, RunMethod
+from synapse_sdk.utils.storage import get_pathlib
+
+
+class ExportParams(BaseModel):
+    storage: int
+    save_original_file: bool = True
+    path: str
+    ground_truth_dataset_version: int
+    filter: dict
+
+    @field_validator('storage')
+    @staticmethod
+    def check_storage_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_storage(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get storage from Synapse backend.'))
+        return value
+
+    @field_validator('ground_truth_dataset_version')
+    @staticmethod
+    def check_ground_truth_dataset_version_exists(value, info):
+        action = info.context['action']
+        client = action.client
+        try:
+            client.get_ground_truth_version(value)
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset version.'))
+        return value
+
+
+@register_action
+class ExportAction(Action):
+    name = 'export'
+    category = PluginCategory.EXPORT
+    method = RunMethod.JOB
+    params_model = ExportParams
+    progress_categories = {
+        'dataset_conversion': {
+            'proportion': 100,
+        }
+    }
+
+    def get_dataset(self, results):
+        """Get dataset for export."""
+        for result in results:
+            yield {
+                'data': result['data'],
+                'files': result['data_unit']['files'],
+                'id': result['ground_truth'],
+            }
+
+    def get_filtered_results(self):
+        """Get filtered ground truth."""
+        ground_truth_dataset_version = self.params['ground_truth_dataset_version']
+        self.params['filter']['ground_truth_dataset'] = self.client.get_ground_truth_version(
+            ground_truth_dataset_version
+        )['ground_truth_dataset']
+        filters = {'expand': 'data', **self.params['filter']}
+
+        try:
+            gt_dataset_events_list = self.client.list_ground_truth_events(params=filters, list_all=True)
+            results = gt_dataset_events_list[0]
+            count = gt_dataset_events_list[1]
+        except ClientError:
+            raise PydanticCustomError('client_error', _('Unable to get Ground Truth dataset.'))
+        return results, count
+
+    def start(self):
+        self.params['results'], self.params['count'] = self.get_filtered_results()
+        dataset = self.get_dataset(self.params['results'])
+
+        storage = self.client.get_storage(self.params['storage'])
+        pathlib_cwd = get_pathlib(storage, self.params['path'])
+        return self.entrypoint(self.run, dataset, pathlib_cwd, **self.params)

synapse_sdk-1.0.0a30/synapse_sdk/plugins/categories/export/actions/utils.py

@@ -0,0 +1,5 @@
+from pathlib import Path
+
+
+def get_original_file_path(files):
+    return Path(next(iter(files.values()))['meta']['path_original'])

synapse_sdk-1.0.0a30/synapse_sdk/plugins/categories/export/templates/config.yaml

@@ -0,0 +1,3 @@
+actions:
+  export:
+    entrypoint: plugin.export.export

synapse_sdk-1.0.0a30/synapse_sdk/plugins/categories/export/templates/plugin/export.py

@@ -0,0 +1,126 @@
+import json
+
+import requests
+
+from synapse_sdk.plugins.categories.export.actions.utils import get_original_file_path
+
+
+def export(run, input_dataset, path_root, **params):
+    """Executes the export task.
+
+    Args:
+        run : Execution object
+        input_dataset (generator):
+            - data (dict): dm_schema_data information.
+            - files (dict): File information. Includes file URL, original file path, metadata, etc.
+            - id (int): ground_truth ID
+        path_root : Save path
+        **params: Additional parameters
+
+    Returns:
+        dict: Result
+    """
+
+    path_root.mkdir(parents=True, exist_ok=True)
+    run.log_message('Starting export process.')
+
+    # results: Contains all information fetched through the list API.
+    results = params.get('results', [])
+
+    save_original_file = params.get('save_original_file')
+    errors_json_file_list = []
+    errors_original_file_list = []
+
+    # Path to save JSON files
+    json_output_path = path_root / 'json'
+    json_output_path.mkdir(parents=True, exist_ok=True)
+
+    # Path to save original files
+    origin_files_output_path = path_root / 'origin_files'
+    origin_files_output_path.mkdir(parents=True, exist_ok=True)
+
+    total = len(results)
+    for no, input_data in enumerate(input_dataset):
+        run.set_progress(no, total, category='dataset_conversion')
+        preprocessed_data = before_convert(input_data)
+        converted_data = convert_data(preprocessed_data)
+        final_data = after_convert(converted_data)
+
+        # Call if original file extraction is needed
+        if save_original_file:
+            save_original_file(final_data, origin_files_output_path, errors_original_file_list)
+
+        # Extract data as JSON files
+        save_as_json(final_data, json_output_path, errors_json_file_list)
+
+    run.log_message('Saving converted dataset.')
+    run.end_log()
+
+    # Save error list files
+    if len(errors_json_file_list) > 0 or len(errors_original_file_list) > 0:
+        export_error_file = {'json_file_name': errors_json_file_list, 'origin_file_name': errors_original_file_list}
+        with (path_root / 'error_file_list.json').open('w', encoding='utf-8') as f:
+            json.dump(export_error_file, f, indent=4, ensure_ascii=False)
+
+    return {'export_path': path_root}
+
+
+def convert_data(data):
+    """Converts the data."""
+    return data
+
+
+def before_convert(data):
+    """Preprocesses the data before conversion."""
+    return data
+
+
+def after_convert(data):
+    """Post-processes the data after conversion."""
+    return data
+
+
+def save_original_file(result, base_path, error_file_list):
+    """Saves the original file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    file_url = next(iter(result['files'].values()))['url']
+    file_name = get_original_file_path(result['files']).name
+    response = requests.get(file_url)
+    try:
+        with (base_path / file_name).open('wb') as file:
+            file.write(response.content)
+    except Exception as e:
+        error_file_list.append([file_name, str(e)])
+
+    return base_path
+
+
+def save_as_json(result, base_path, error_file_list):
+    """Saves the data as a JSON file.
+
+    Args:
+        result (dict): Result data
+        base_path (Path): Save path
+        error_file_list (list): List of error files
+
+    Returns:
+        base_path (str): Save path
+    """
+    # Default save file name: original file name
+    file_name = get_original_file_path(result['files']).stem
+    json_data = result['data']
+    try:
+        with (base_path / f'{file_name}.json').open('w', encoding='utf-8') as f:
+            json.dump(json_data, f, indent=4, ensure_ascii=False)
+    except Exception as e:
+        error_file_list.append([f'{file_name}.json', str(e)])
+
+    return base_path

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/plugins/categories/neural_net/actions/train.py

@@ -18,8 +18,13 @@ from synapse_sdk.utils.pydantic.validators import non_blank
 
 class TrainRun(Run):
     def log_metric(self, category, key, value, **metrics):
+        # TODO validate input via plugin config
         self.log('metric', {'category': category, 'key': key, 'value': value, 'metrics': metrics})
 
+    def log_visualization(self, category, group, image, **meta):
+        # TODO validate input via plugin config
+        self.log('visualization', {'category': category, 'group': group, **meta}, file=image)
+
 
 class Hyperparameter(BaseModel):
     batch_size: int

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/plugins/categories/neural_net/templates/config.yaml

@@ -8,6 +8,9 @@ actions:
       epoch:
         - loss
         - miou
+    hyperparameters:
+      ui_schema: |
+        Dumped FormKit Schema for hyperparameters
   deployment:
     entrypoint: plugin.inference.MockNetInference
   inference:

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/plugins/models.py

@@ -113,8 +113,8 @@ class Run:
     def set_progress(self, current, total, category=''):
         self.logger.set_progress(current, total, category)
 
-    def log(self, event, data):
-        self.logger.log(event, data)
+    def log(self, event, data, file=None):
+        self.logger.log(event, data, file=file)
 
     def log_message(self, message, context=Context.INFO.value):
         self.logger.log('message', {'context': context, 'content': message})

{synapse_sdk-1.0.0a28 → synapse_sdk-1.0.0a30}/synapse_sdk/utils/file.py

@@ -1,6 +1,8 @@
 import asyncio
+import base64
 import hashlib
 import json
+import mimetypes
 import operator
 import zipfile
 from functools import reduce
@@ -182,3 +184,36 @@ def get_temp_path(sub_path=None):
     if sub_path:
         path = path / sub_path
     return path
+
+
+def convert_file_to_base64(file_path):
+    """
+    Convert a file to base64 using pathlib.
+
+    Args:
+        file_path (str): Path to the file to convert
+
+    Returns:
+        str: Base64 encoded string of the file contents
+    """
+    # Convert string path to Path object
+    path = Path(file_path)
+
+    try:
+        # Read binary content of the file
+        binary_content = path.read_bytes()
+
+        # Convert to base64
+        base64_encoded = base64.b64encode(binary_content).decode('utf-8')
+
+        # Get the MIME type of the file
+        mime_type, _ = mimetypes.guess_type(path)
+        assert mime_type is not None, 'MIME type cannot be guessed'
+
+        # Convert bytes to string for readable output
+        return f'data:{mime_type};base64,{base64_encoded}'
+
+    except FileNotFoundError:
+        raise FileNotFoundError(f'File not found: {file_path}')
+    except Exception as e:
+        raise Exception(f'Error converting file to base64: {str(e)}')

synapse_sdk-1.0.0a30/synapse_sdk/utils/storage/__init__.py

@@ -0,0 +1,20 @@
+from urllib.parse import urlparse
+
+from synapse_sdk.i18n import gettext as _
+from synapse_sdk.utils.storage.registry import STORAGE_PROVIDERS
+
+
+def get_storage(connection_param: str | dict):
+    storage_scheme = None
+    if isinstance(connection_param, dict):
+        storage_scheme = connection_param['provider']
+    else:
+        storage_scheme = urlparse(connection_param).scheme
+
+    assert storage_scheme in STORAGE_PROVIDERS.keys(), _('지원하지 않는 저장소입니다.')
+    return STORAGE_PROVIDERS[storage_scheme](connection_param)
+
+
+def get_pathlib(storage_config, path_root):
+    storage_class = get_storage(storage_config)
+    return storage_class.get_pathlib(path_root)

synapse_sdk-1.0.0a30/synapse_sdk/utils/storage/providers/__init__.py

@@ -0,0 +1,42 @@
+from urllib.parse import parse_qs, urlparse
+
+
+class BaseStorage:
+    url = None
+    options = None
+    OPTION_CASTS = {}
+
+    def __init__(self, connection_params: str | dict):
+        self.url = None
+
+        if isinstance(connection_params, dict):
+            self.query_params = connection_params['configuration']
+        else:
+            self.url = urlparse(connection_params)
+            self.query_params = self.url_querystring_to_dict()
+
+    def url_querystring_to_dict(self):
+        query_string = self.url.query
+
+        query_dict = parse_qs(query_string)
+
+        for key, value in query_dict.items():
+            if len(value) == 1:
+                query_dict[key] = value[0]
+
+        return {
+            key: self.OPTION_CASTS[key](value) if key in self.OPTION_CASTS else value
+            for key, value in query_dict.items()
+        }
+
+    def upload(self, source, target):
+        raise NotImplementedError
+
+    def exists(self, target):
+        raise NotImplementedError
+
+    def get_url(self, target):
+        raise NotImplementedError
+
+    def get_pathlib(self, path):
+        raise NotImplementedError

synapse_sdk-1.0.0a30/synapse_sdk/utils/storage/providers/gcp.py

@@ -0,0 +1,13 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class GCPStorage(BaseStorage):
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = UPath(f'gs://{self.query_params["bucket_name"]}', token=self.query_params['credentials'])
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk-1.0.0a30/synapse_sdk/utils/storage/providers/s3.py

@@ -0,0 +1,43 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class S3Storage(BaseStorage):
+    ENDPOINT_URL = 'https://s3.amazonaws.com'
+    DEFAULT_REGION = 'us-east-1'
+
+    def __init__(self, url):
+        super().__init__(url)
+
+        self.upath = self._get_upath()
+
+    def _get_upath(self):
+        upath_kwargs = {
+            'key': self.query_params['access_key'],
+            'secret': self.query_params['secret_key'],
+            'client_kwargs': {'region_name': self.query_params.get('region_name')},
+        }
+
+        if self.query_params.get('endpoint_url'):
+            upath_kwargs['endpoint_url'] = self.query_params['endpoint_url']
+
+        return UPath(
+            f's3://{self.query_params["bucket_name"]}',
+            **upath_kwargs,
+        )
+
+    def upload(self, source, target):
+        with open(source, 'rb') as file:
+            self.upath.write_text(file.read(), target)
+
+        return self.get_url(target)
+
+    def exists(self, target):
+        return self.upath.exists(target)
+
+    def get_url(self, target):
+        return str(self.upath.joinuri(target))
+
+    def get_pathlib(self, path):
+        return self.upath.joinuri(path)

synapse_sdk-1.0.0a30/synapse_sdk/utils/storage/providers/sftp.py

@@ -0,0 +1,16 @@
+from upath import UPath
+
+from synapse_sdk.utils.storage.providers import BaseStorage
+
+
+class SFTPStorage(BaseStorage):
+    def get_pathlib(self, path):
+        credentials = self.query_params['params']
+        host = self.query_params['host']
+        root_path = self.query_params['root_path']
+
+        username = credentials['username']
+        password = credentials['password']
+        if path == '/':
+            path = ''
+        return UPath(f'sftp://{host}', username=username, password=password) / root_path / path

synapse_sdk-1.0.0a30/synapse_sdk/utils/storage/registry.py

@@ -0,0 +1,11 @@
+from synapse_sdk.utils.storage.providers.gcp import GCPStorage
+from synapse_sdk.utils.storage.providers.s3 import S3Storage
+from synapse_sdk.utils.storage.providers.sftp import SFTPStorage
+
+STORAGE_PROVIDERS = {
+    's3': S3Storage,
+    'amazon_s3': S3Storage,
+    'minio': S3Storage,
+    'gcp': GCPStorage,
+    'sftp': SFTPStorage,
+}